5d445f2002-03-20Martin Nilsson // // This file needs to support old pikes that e.g. don't understand // "#pike". Some of them fail when they see an unknown cpp directive. // // #pike __REAL_VERSION__ //
916c022002-06-28Martin Nilsson 
e1e2d12003-06-18Martin Stjernholm //! This module parses and tokenizes Pike source code.
a20af62000-09-26Fredrik Hübinette (Hubbe) 
74c57a2000-07-14Andreas Lange inherit "C.pmod";
216bb92014-08-10Martin Nilsson array(string) low_split(string data, void|mapping(string:string) state)
19cbfb2005-03-24Martin Nilsson { if(state && state->remains)
216bb92014-08-10Martin Nilsson  data = (string)m_delete(state, "remains") + data; // Cast to string above to work around old Pike 7.0 bug.
19cbfb2005-03-24Martin Nilsson  array ret; string rem; [ret, rem] = Parser._parser._Pike.tokenize(data); if(sizeof(rem)) {
9e19302010-02-14Martin Nilsson  if(rem[0]=='"') throw(UnterminatedStringError(ret*"", rem));
19cbfb2005-03-24Martin Nilsson  if(state) state->remains=rem; } return ret; }
ba7b0a2001-01-18David Norlin 
916c022002-06-28Martin Nilsson //! Splits the @[data] string into an array of tokens. An additional //! element with a newline will be added to the resulting array of //! tokens. If the optional argument @[state] is provided the split //! function is able to pause and resume splitting inside #"" and //! /**/ tokens. The @[state] argument should be an initially empty //! mapping, in which split will store its state between successive //! calls.
216bb92014-08-10Martin Nilsson array(string) split(string data, void|mapping(string:string) state) {
19cbfb2005-03-24Martin Nilsson  array r = low_split(data, state); array new = ({}); for(int i; i<sizeof(r); i++)
c7bca92005-04-10Martin Nilsson  if(r[i][..1]=="//" && r[i][-1]=='\n') new += ({ r[i][..<1], "\n" });
19cbfb2005-03-24Martin Nilsson  else new += ({ r[i] }); if(sizeof(new) && (< "\n", " " >)[new[-1]]) new[-1] += "\n"; else new += ({ "\n" }); return new;
74c57a2000-07-14Andreas Lange }
e0f1792007-06-01Martin Bähr  class UnterminatedStringError //! Error thrown when an unterminated string token is encountered. { inherit Error.Generic; constant error_type = "unterminated_string"; constant is_unterminated_string_error = 1; string err_str; //! The string that failed to be tokenized
9e19302010-02-14Martin Nilsson  protected void create(string pre, string post)
e0f1792007-06-01Martin Bähr  {
9e19302010-02-14Martin Nilsson  int line = String.count(pre, "\n")+1; err_str = pre+post; if( sizeof(post) > 100 ) ::create(sprintf("Unterminated string %O[%d] at line %d\n", post[..100], sizeof(post)-100, line)); else ::create(sprintf("Unterminated string %O at line %d\n", post, line));
e0f1792007-06-01Martin Bähr  } }