0290852005-01-02Martin Nilsson /* array(array(string),string) split( string x ) * Returns an array with Pike-level tokens and the remainder (a * partial token), if any.
55355a2004-10-05Martin Nilsson  */ #include "global.h" #include "config.h" #include "pike_macros.h" #include "object.h" #include "constants.h" #include "interpret.h" #include "svalue.h" #include "threads.h" #include "array.h" #include "pike_error.h" #include "operators.h" #include "builtin_functions.h" #include "module_support.h" #include "mapping.h" #include "stralloc.h" #include "program_id.h" #include <ctype.h>
b5f91a2017-03-18Martin Nilsson #include "whitespace.h"
55355a2004-10-05Martin Nilsson  #include "parser.h"
493b3a2010-05-11Henrik Grubbström (Grubba) /*! @module Parser */ /*! @module _parser */ /*! @module _Pike *! *! Low-level helpers for @[Parser.Pike]. *! *! @note *! You probably want to use @[Parser.Pike] instead of this module. *! *! @seealso *! @[Parser.Pike], @[_C]. */
01b9212016-01-12Per Hedbor inline static int m_isidchar( unsigned int x )
55355a2004-10-05Martin Nilsson {
a0d3fb2017-03-18Martin Nilsson  if( (x>='0' && x<='9') ) return 0; return isidchar(x);
55355a2004-10-05Martin Nilsson } #define PUSH_TOKEN push_token0 #define TOKENIZE tokenize0 #define CHAR p_wchar0
b5f91a2017-03-18Martin Nilsson #define m_isidchar2(X) isidchar(X) #define SPACECASE SPACECASE8
55355a2004-10-05Martin Nilsson #include "pike_tokenizer.h"
b5f91a2017-03-18Martin Nilsson #undef SPACECASE #undef m_isidchar2
55355a2004-10-05Martin Nilsson #undef TOKENIZE #undef CHAR #undef PUSH_TOKEN #define PUSH_TOKEN push_token1 #define TOKENIZE tokenize1 #define CHAR p_wchar1
b5f91a2017-03-18Martin Nilsson #define m_isidchar2(X) wide_isidchar(X) #define SPACECASE SPACECASE16
55355a2004-10-05Martin Nilsson #include "pike_tokenizer.h"
b5f91a2017-03-18Martin Nilsson #undef SPACECASE #undef m_isidchar2
55355a2004-10-05Martin Nilsson #undef TOKENIZE #undef CHAR #undef PUSH_TOKEN #define PUSH_TOKEN push_token2 #define TOKENIZE tokenize2 #define CHAR p_wchar2
b5f91a2017-03-18Martin Nilsson #define m_isidchar2(X) wide_isidchar(X) #define SPACECASE SPACECASE16
55355a2004-10-05Martin Nilsson #include "pike_tokenizer.h"
b5f91a2017-03-18Martin Nilsson #undef SPACECASE #undef m_isidchar2
55355a2004-10-05Martin Nilsson #undef TOKENIZE #undef CHAR #undef PUSH_TOKEN static void do_free_arrayptr( struct array **x ) { free_array( *x );
13670c2015-05-25Martin Nilsson }
55355a2004-10-05Martin Nilsson 
493b3a2010-05-11Henrik Grubbström (Grubba) /*! @decl array(array(string)|string) tokenize(string code) *! *! Tokenize a string of Pike tokens. *! *! @returns *! Returns an array with Pike-level tokens and the remainder (a *! partial token), if any.
18b7952005-03-24Henrik Grubbström (Grubba)  */
55355a2004-10-05Martin Nilsson static void f_tokenize( INT32 args ) {
00936a2005-03-16Martin Nilsson  struct array *res;
18b7952005-03-24Henrik Grubbström (Grubba)  struct pike_string *left_s = NULL; /* Make gcc happy. */ struct pike_string *data;
55355a2004-10-05Martin Nilsson  int left; ONERROR tmp;
18b7952005-03-24Henrik Grubbström (Grubba)  get_all_args("tokenize", args, "%W", &data);
55355a2004-10-05Martin Nilsson 
18b7952005-03-24Henrik Grubbström (Grubba)  if(!data->len)
00936a2005-03-16Martin Nilsson  { pop_n_elems(args);
ae35132005-05-05Martin Nilsson  push_empty_array(); push_empty_string();
00936a2005-03-16Martin Nilsson  f_aggregate(2); return; } res = allocate_array_no_init( 0, 128 );
55355a2004-10-05Martin Nilsson  SET_ONERROR(tmp, do_free_arrayptr, &res);
13670c2015-05-25Martin Nilsson 
18b7952005-03-24Henrik Grubbström (Grubba)  switch(data->size_shift)
55355a2004-10-05Martin Nilsson  { case 0:
18b7952005-03-24Henrik Grubbström (Grubba)  left = tokenize0(&res, STR0(data), data->len); left_s = make_shared_binary_string0(STR0(data)+left, data->len-left);
55355a2004-10-05Martin Nilsson  break; case 1:
18b7952005-03-24Henrik Grubbström (Grubba)  left = tokenize1(&res, STR1(data), data->len); left_s = make_shared_binary_string1(STR1(data)+left, data->len-left);
55355a2004-10-05Martin Nilsson  break; case 2:
18b7952005-03-24Henrik Grubbström (Grubba)  left = tokenize2(&res,STR2(data), data->len);
da6d082005-03-28Martin Nilsson  left_s = make_shared_binary_string2(STR2(data)+left, data->len-left);
55355a2004-10-05Martin Nilsson  break; }
18b7952005-03-24Henrik Grubbström (Grubba)  UNSET_ONERROR(tmp);
55355a2004-10-05Martin Nilsson  pop_n_elems(args);
24e1fd2005-04-26Henrik Grubbström (Grubba)  if (!res->size) { free_array(res);
ae35132005-05-05Martin Nilsson  push_empty_array();
24e1fd2005-04-26Henrik Grubbström (Grubba)  }
ae35132005-05-05Martin Nilsson  else push_array(res);
55355a2004-10-05Martin Nilsson  push_string( left_s ); f_aggregate( 2 ); }
493b3a2010-05-11Henrik Grubbström (Grubba) /*! @endmodule */ /*! @endmodule */ /*! @endmodule */
55355a2004-10-05Martin Nilsson  void init_parser_pike() {
da6d082005-03-28Martin Nilsson  ADD_FUNCTION2("tokenize", f_tokenize, tFunc(tStr,tArr(tOr(tArr(tStr),tStr))), 0, 0);
55355a2004-10-05Martin Nilsson } void exit_parser_pike() { }