0290852005-01-02Martin Nilsson /* array(array(string),string) split( string x ) * Returns an array with Pike-level tokens and the remainder (a * partial token), if any.
55355a2004-10-05Martin Nilsson  */ #include "global.h" #include "config.h" #include "pike_macros.h" #include "object.h" #include "constants.h" #include "interpret.h" #include "svalue.h" #include "threads.h" #include "array.h" #include "pike_error.h" #include "operators.h" #include "builtin_functions.h" #include "module_support.h" #include "mapping.h" #include "stralloc.h" #include "program_id.h" #include "block_alloc.h" #include <ctype.h> #include "parser.h" INLINE static int m_isidchar( unsigned int x ) { if( (x >= 'a' && x <= 'z') || (x>='A' && x<='Z') || x>128 || x == '_') return 1; return 0; } INLINE static int m_isidchar2( unsigned int x ) { if( (x >= 'a' && x <= 'z') || (x>='A' && x<='Z') || x>128 || x=='_' || (x>='0'&&x<='9')||x=='$') return 1; return 0; } #define PUSH_TOKEN push_token0 #define TOKENIZE tokenize0 #define CHAR p_wchar0 #include "pike_tokenizer.h" #undef TOKENIZE #undef CHAR #undef PUSH_TOKEN #define PUSH_TOKEN push_token1 #define TOKENIZE tokenize1 #define CHAR p_wchar1 #include "pike_tokenizer.h" #undef TOKENIZE #undef CHAR #undef PUSH_TOKEN #define PUSH_TOKEN push_token2 #define TOKENIZE tokenize2 #define CHAR p_wchar2 #include "pike_tokenizer.h" #undef TOKENIZE #undef CHAR #undef PUSH_TOKEN static void do_free_arrayptr( struct array **x ) { free_array( *x ); }
bdda382010-05-08Martin Stjernholm /* @decl array(array(string)|string) tokenize(string code) * * Tokenize a string of Pike tokens. * * @returns * Returns an array with Pike-level tokens and the remainder (a * partial token), if any.
18b7952005-03-24Henrik Grubbström (Grubba)  */
55355a2004-10-05Martin Nilsson static void f_tokenize( INT32 args ) {
00936a2005-03-16Martin Nilsson  struct array *res;
18b7952005-03-24Henrik Grubbström (Grubba)  struct pike_string *left_s = NULL; /* Make gcc happy. */ struct pike_string *data;
55355a2004-10-05Martin Nilsson  int left; ONERROR tmp;
18b7952005-03-24Henrik Grubbström (Grubba)  get_all_args("tokenize", args, "%W", &data);
55355a2004-10-05Martin Nilsson 
18b7952005-03-24Henrik Grubbström (Grubba)  if(!data->len)
00936a2005-03-16Martin Nilsson  { pop_n_elems(args);
ae35132005-05-05Martin Nilsson  push_empty_array(); push_empty_string();
00936a2005-03-16Martin Nilsson  f_aggregate(2); return; } res = allocate_array_no_init( 0, 128 );
55355a2004-10-05Martin Nilsson  SET_ONERROR(tmp, do_free_arrayptr, &res);
18b7952005-03-24Henrik Grubbström (Grubba)  switch(data->size_shift)
55355a2004-10-05Martin Nilsson  { case 0:
18b7952005-03-24Henrik Grubbström (Grubba)  left = tokenize0(&res, STR0(data), data->len); left_s = make_shared_binary_string0(STR0(data)+left, data->len-left);
55355a2004-10-05Martin Nilsson  break; case 1:
18b7952005-03-24Henrik Grubbström (Grubba)  left = tokenize1(&res, STR1(data), data->len); left_s = make_shared_binary_string1(STR1(data)+left, data->len-left);
55355a2004-10-05Martin Nilsson  break; case 2:
18b7952005-03-24Henrik Grubbström (Grubba)  left = tokenize2(&res,STR2(data), data->len);
da6d082005-03-28Martin Nilsson  left_s = make_shared_binary_string2(STR2(data)+left, data->len-left);
55355a2004-10-05Martin Nilsson  break; #ifdef PIKE_DEBUG default:
18b7952005-03-24Henrik Grubbström (Grubba)  Pike_error("Unknown shift size %d.\n", data->size_shift);
55355a2004-10-05Martin Nilsson #endif }
18b7952005-03-24Henrik Grubbström (Grubba)  UNSET_ONERROR(tmp);
55355a2004-10-05Martin Nilsson  pop_n_elems(args);
24e1fd2005-04-26Henrik Grubbström (Grubba)  if (!res->size) { free_array(res);
ae35132005-05-05Martin Nilsson  push_empty_array();
24e1fd2005-04-26Henrik Grubbström (Grubba)  }
ae35132005-05-05Martin Nilsson  else push_array(res);
55355a2004-10-05Martin Nilsson  push_string( left_s ); f_aggregate( 2 ); } void init_parser_pike() {
da6d082005-03-28Martin Nilsson  ADD_FUNCTION2("tokenize", f_tokenize, tFunc(tStr,tArr(tOr(tArr(tStr),tStr))), 0, 0);
55355a2004-10-05Martin Nilsson } void exit_parser_pike() { }