1
  
2
  
3
  
4
  
5
  
6
  
7
  
8
  
9
  
10
  
11
  
12
  
13
  
14
  
15
  
16
  
17
  
18
  
19
  
20
  
21
  
22
  
23
  
24
  
25
  
26
  
27
  
28
  
29
  
30
  
31
  
32
  
33
  
34
  
35
  
36
  
37
  
38
  
39
  
40
  
41
  
42
  
43
  
44
  
45
  
46
  
47
  
48
  
49
  
50
  
51
  
52
  
53
  
54
  
55
  
56
  
57
  
58
  
59
  
60
  
61
  
62
  
63
  
64
  
65
  
66
  
67
  
68
  
69
  
70
  
71
  
72
  
73
  
74
  
75
  
76
  
77
  
78
  
79
  
80
  
81
  
82
  
83
  
84
  
85
  
86
  
87
  
88
  
89
  
90
  
91
  
92
  
93
  
94
  
95
  
96
  
97
  
98
  
99
  
100
  
101
  
102
  
103
  
104
  
105
  
106
  
107
  
108
  
109
  
110
  
111
  
112
  
113
  
114
  
115
  
116
  
117
  
118
  
119
  
120
  
121
  
122
  
/*! array(array(string),string) split( string x ) 
 *! Returns an array with Pike-level tokens and the remainder (a 
 *! partial token), if any. 
 */ 
#include "global.h" 
#include "config.h" 
 
#include "pike_macros.h" 
#include "object.h" 
#include "constants.h" 
#include "interpret.h" 
#include "svalue.h" 
#include "threads.h" 
#include "array.h" 
#include "pike_error.h" 
#include "operators.h" 
#include "builtin_functions.h" 
#include "module_support.h" 
#include "mapping.h" 
#include "stralloc.h" 
#include "program_id.h" 
#include "block_alloc.h" 
#include <ctype.h> 
 
#include "parser.h" 
 
INLINE static int m_isidchar( unsigned int x ) 
{ 
  if( (x >= 'a' && x <= 'z') || (x>='A' && x<='Z') || x>128 || x == '_') 
    return 1; 
  return 0; 
} 
 
INLINE static int m_isidchar2( unsigned int x ) 
{ 
  if( (x >= 'a' && x <= 'z') || (x>='A' && x<='Z') || x>128 || x=='_' || 
      (x>='0'&&x<='9')||x=='$') 
    return 1; 
  return 0; 
} 
 
 
#define PUSH_TOKEN push_token0 
#define TOKENIZE     tokenize0 
#define CHAR       p_wchar0 
#include "pike_tokenizer.h" 
#undef TOKENIZE 
#undef CHAR 
#undef PUSH_TOKEN 
 
#define PUSH_TOKEN push_token1 
#define TOKENIZE   tokenize1 
#define CHAR       p_wchar1 
#include "pike_tokenizer.h" 
#undef TOKENIZE 
#undef CHAR 
#undef PUSH_TOKEN 
 
#define PUSH_TOKEN push_token2 
#define TOKENIZE   tokenize2 
#define CHAR       p_wchar2 
#include "pike_tokenizer.h" 
#undef TOKENIZE 
#undef CHAR 
#undef PUSH_TOKEN 
 
static void do_free_arrayptr( struct array **x ) 
{ 
  free_array( *x ); 
}                           
 
static void f_tokenize( INT32 args ) 
{ 
  struct array *res = allocate_array_no_init( 0, 128 ); 
  struct pike_string *left_s = 0; /* Make gcc happy. */ 
  int left; 
  ONERROR tmp; 
 
  if( Pike_sp[-1].type != PIKE_T_STRING ) 
    Pike_error("Expected string argument\n"); 
 
  SET_ONERROR(tmp, do_free_arrayptr, &res); 
   
  switch( Pike_sp[-1].u.string->size_shift ) 
  { 
    case 0: 
      left=tokenize0(&res,(p_wchar0*)Pike_sp[-1].u.string->str,Pike_sp[-1].u.string->len); 
      left_s = make_shared_binary_string0( (p_wchar0*)Pike_sp[-1].u.string->str+left, 
                                           Pike_sp[-1].u.string->len-left); 
      break; 
    case 1: 
      left=tokenize1(&res,(p_wchar1*)Pike_sp[-1].u.string->str,Pike_sp[-1].u.string->len); 
      left_s = make_shared_binary_string1( (p_wchar1*)Pike_sp[-1].u.string->str+left, 
                                           Pike_sp[-1].u.string->len-left); 
      break; 
    case 2: 
      left=tokenize2(&res,(p_wchar2*)Pike_sp[-1].u.string->str,Pike_sp[-1].u.string->len); 
      left_s = make_shared_binary_string1( (p_wchar1*)Pike_sp[-1].u.string->str+left, 
                                           Pike_sp[-1].u.string->len-left); 
      break; 
#ifdef PIKE_DEBUG 
    default: 
      Pike_error("Unknown shift size %d.\n", Pike_sp[-1].u.string->size_shift); 
#endif 
  } 
  pop_n_elems(args); 
  push_array(res); 
  push_string( left_s ); 
  f_aggregate( 2 ); 
} 
 
 
void init_parser_pike() 
{ 
  ADD_FUNCTION("tokenize",f_tokenize,tFunc(tStr,tArr(tStr)),0); 
} 
 
void exit_parser_pike() 
{ 
}