pike.git/ src/ modules/ Parser/ pike.c > a0d3fb6800089689aca37c50fc83ef500a66c926 [Annotate]

/* array(array(string),string) split( string x ) 

 * Returns an array with Pike-level tokens and the remainder (a 

 * partial token), if any. 

 */ 

#include "global.h" 

#include "config.h" 

#include "pike_macros.h" 

#include "object.h" 

#include "constants.h" 

#include "interpret.h" 

#include "svalue.h" 

#include "threads.h" 

#include "array.h" 

#include "pike_error.h" 

#include "operators.h" 

#include "builtin_functions.h" 

#include "module_support.h" 

#include "mapping.h" 

#include "stralloc.h" 

#include "program_id.h" 

#include <ctype.h> 

#include "parser.h" 

/*! @module Parser 

 */ 

/*! @module _parser 

 */ 

/*! @module _Pike 

 *! 

 *! Low-level helpers for @[Parser.Pike]. 

 *! 

 *! @note 

 *!   You probably want to use @[Parser.Pike] instead of this module. 

 *! 

 *! @seealso 

 *!   @[Parser.Pike], @[_C]. 

 */ 

inline static int m_isidchar( unsigned int x ) 

{ 

  if( (x>='0' && x<='9') ) return 0; 

  return isidchar(x); 

} 

#define m_isidchar2(X) isidchar(X) 

#define PUSH_TOKEN push_token0 

#define TOKENIZE     tokenize0 

#define CHAR       p_wchar0 

#include "pike_tokenizer.h" 

#undef TOKENIZE 

#undef CHAR 

#undef PUSH_TOKEN 

#define PUSH_TOKEN push_token1 

#define TOKENIZE   tokenize1 

#define CHAR       p_wchar1 

#include "pike_tokenizer.h" 

#undef TOKENIZE 

#undef CHAR 

#undef PUSH_TOKEN 

#define PUSH_TOKEN push_token2 

#define TOKENIZE   tokenize2 

#define CHAR       p_wchar2 

#include "pike_tokenizer.h" 

#undef TOKENIZE 

#undef CHAR 

#undef PUSH_TOKEN 

static void do_free_arrayptr( struct array **x ) 

{ 

  free_array( *x ); 

} 

/*! @decl array(array(string)|string) tokenize(string code) 

 *! 

 *!   Tokenize a string of Pike tokens. 

 *! 

 *! @returns 

 *!   Returns an array with Pike-level tokens and the remainder (a 

 *!   partial token), if any. 

 */ 

static void f_tokenize( INT32 args ) 

{ 

  struct array *res; 

  struct pike_string *left_s = NULL; /* Make gcc happy. */ 

  struct pike_string *data; 

  int left; 

  ONERROR tmp; 

  get_all_args("tokenize", args, "%W", &data); 

  if(!data->len) 

  { 

    pop_n_elems(args); 

    push_empty_array(); 

    push_empty_string(); 

    f_aggregate(2); 

    return; 

  } 

  res = allocate_array_no_init( 0, 128 ); 

  SET_ONERROR(tmp, do_free_arrayptr, &res); 

  switch(data->size_shift) 

  { 

    case 0: 

      left = tokenize0(&res, STR0(data), data->len); 

      left_s = make_shared_binary_string0(STR0(data)+left, data->len-left); 

      break; 

    case 1: 

      left = tokenize1(&res, STR1(data), data->len); 

      left_s = make_shared_binary_string1(STR1(data)+left, data->len-left); 

      break; 

    case 2: 

      left = tokenize2(&res,STR2(data), data->len); 

      left_s = make_shared_binary_string2(STR2(data)+left, data->len-left); 

      break; 

  } 

  UNSET_ONERROR(tmp); 

  pop_n_elems(args); 

  if (!res->size) { 

    free_array(res); 

    push_empty_array(); 

  } 

  else 

    push_array(res); 

  push_string( left_s ); 

  f_aggregate( 2 ); 

} 

/*! @endmodule 

 */ 

/*! @endmodule 

 */ 

/*! @endmodule 

 */ 

void init_parser_pike() 

{ 

  ADD_FUNCTION2("tokenize", f_tokenize, 

                tFunc(tStr,tArr(tOr(tArr(tStr),tStr))), 0, 0); 

} 

void exit_parser_pike() 

{ 

}