Branch: Tag:

2001-03-06

2001-03-06 16:40:19 by David Norlin <norlin@roxen.com>

New error reporting system with better line info.

Rev: lib/modules/Tools.pmod/AutoDoc.pmod/CExtractor.pmod:1.10
Rev: lib/modules/Tools.pmod/AutoDoc.pmod/DocParser.pmod:1.24
Rev: lib/modules/Tools.pmod/AutoDoc.pmod/PikeExtractor.pmod:1.8
Rev: lib/modules/Tools.pmod/AutoDoc.pmod/PikeObjects.pmod:1.10
Rev: lib/modules/Tools.pmod/AutoDoc.pmod/PikeParser.pike:1.13
Rev: lib/modules/Tools.pmod/AutoDoc.pmod/ProcessXML.pmod:1.9
Rev: lib/modules/Tools.pmod/AutoDoc.pmod/debug.h:1.1
Rev: lib/modules/Tools.pmod/AutoDoc.pmod/module.pmod:1.4

3:   //========================================================================      inherit "module.pmod"; -  - #define DEB werror("###DocParser.Pike: %d\n", __LINE__); -  - static void parseError(string s, mixed ... args) { -  s = sprintf(s, @args); -  werror(s+"\n"); -  s = "DocParser error: " + s; -  throw(({ s, 0 })); - } -  +    inherit .PikeObjects;    -  + #include "./debug.h" +    class MetaData {    string type;    string name; // if (type == "class", "module", "endmodule", or "endclass")
104:      mapping(string : multiset(string)) allowGrouping =   (["param" : (< "param" >), +  "index" : (< "index" >),    "member" : (< "member" >),    "type" : (< "type" >),    "value" : (< "value" >),   ]);    -  + static int getKeywordType(string keyword) { +  if (keywordtype[keyword]) +  return keywordtype[keyword]; +  if (strlen(keyword) > 3 && keyword[0..2] == "end") +  return ENDKEYWORD; +  return ERRORKEYWORD; + } +  + static int getTokenType(array(string) | string token) { +  if (arrayp(token)) +  return getKeywordType(token[0]); +  if (!token) +  return ENDTOKEN; +  return TEXTTOKEN; + } +  + static int isSpaceChar(int char) { +  return (< '\t', '\n', ' ' >) [char]; + } +  + static int isKeywordChar(int char) { +  return char >= 'a' && char <= 'z'; + } +  + static array(string) extractKeyword(string line) { +  line += "\0"; +  int i = 0; +  while (i < strlen(line) && isSpaceChar(line[i])) +  ++i; +  if (line[i++] != '@') +  return 0; +  int keywordstart = i; +  while (isKeywordChar(line[i])) +  ++i; +  if (i == keywordstart || line[i] && !isSpaceChar(line[i])) +  return 0; +  string keyword = line[keywordstart .. i - 1]; +  // if (getKeywordType(keyword) == METAKEYWORD) +  return ({ keyword, line[i .. strlen(line) - 2] }); // skippa "\0" ... + } +  + static int allSpaces(string s) { +  for (int i = strlen(s) - 1; i >= 0; --i) +  if (s[i] != ' ' && s[i] != '\t') +  return 0; +  return 1; + } +  + static private class Token ( +  int type, +  string keyword, +  string arg, // the rest of the line, after the keyword +  string text, +  SourcePosition position, + ) { +  string _sprintf() { +  return sprintf("Token(%d, %O, %O, %O, %O)", type, keyword, +  arg, text, position); +  } + } +  + static array(Token) split(string s, SourcePosition pos) { +  s = s - "\r"; +  s = replace(s, "@\n", "@\r"); +  array(string) lines = s / "\n"; +  +  // array holding the number of src lines per line +  array(int) counts = allocate(sizeof(lines)); +  for(int i = 0; i < sizeof(lines); ++i) { +  array(string) a = lines[i] / "@\r"; +  counts[i] = sizeof(a); +  lines[i] = a * ""; +  } +  array(Token) res = ({ }); +  +  string filename = pos->filename; +  string text = 0; +  int textFirstLine; +  int textLastLine; +  +  int curLine = pos->firstline; +  +  for(int i = 0; i < sizeof(lines); ++i) { +  string line = lines[i]; +  array(string) extr = extractKeyword(line); +  if (extr) { +  if (text) +  res += ({ +  Token(TEXTTOKEN, 0, 0, text, +  SourcePosition(filename, textFirstLine, textLastLine)) +  }); +  text = 0; +  res += ({ +  Token(getKeywordType(extr[0]), extr[0], extr[1], 0, +  SourcePosition(filename, curLine, curLine + counts[i] - 1)) +  }); +  } +  else { +  if (allSpaces(line)) +  line = ""; +  if (!text) { +  text = ""; +  textFirstLine = curLine; +  } +  text += line + "\n"; +  textLastLine = curLine; +  } +  curLine += counts[i]; +  } +  if (text) +  res += ({ Token(TEXTTOKEN, 0, 0, text, +  SourcePosition(filename, textFirstLine, textLastLine)) +  }); +  return res; + } +  + static class DocParserClass { +  +  static SourcePosition currentPosition = 0; +  +  static void parseError(string s, mixed ... args) { +  s = sprintf(s, @args); +  werror("DocParser error: %O\n", s); +  throw (AutoDocError(currentPosition, "DocParser", s)); +  } +  +  static array(Token) tokenArr = 0; +  static Token peekToken() { +  Token t = tokenArr[0]; +  currentPosition = t->position || currentPosition; +  return t; +  } +  static Token readToken() { +  Token t = peekToken(); +  tokenArr = tokenArr[1..]; +  return t; +  } +    // argHandlers:   //   // Contains functions that handle keywords with non-standard arg list
132:      static string memberArgHandler(string keyword, string arg) {    // werror("This is the @member arg handler "); -  .PikeParser parser = .PikeParser(arg); +  .PikeParser parser = .PikeParser(arg, currentPosition);    // werror("&&& %O\n", arg);    Type t = parser->parseOrType();    if (!t)
148:      static string elemArgHandler(string keyword, string arg) {    // werror("This is the @elem arg handler\n"); -  .PikeParser parser = .PikeParser(arg); +  .PikeParser parser = .PikeParser(arg, currentPosition);    Type t = parser->parseOrType();    if (!t)    parseError("@elem: expected type, got %O", arg);
180:      static string indexArgHandler(string keyword, string arg) {    // werror("indexArgHandler\n"); -  .PikeParser parser = .PikeParser(arg); +  .PikeParser parser = .PikeParser(arg, currentPosition);    string s = parser->parseLiteral();    if (!s)    parseError("@index: expected identifier, got %O", arg);
189:   }      static string deprArgHandler(string keyword, string arg) { -  .PikeParser parser = .PikeParser(arg); +  .PikeParser parser = .PikeParser(arg, currentPosition);    if (parser->peekToken() == EOF)    return "";    string res = "";
210:      static string typeArgHandler(string keyword, string arg) {    // werror("This is the @type arg handler "); -  .PikeParser parser = .PikeParser(arg); +  .PikeParser parser = .PikeParser(arg, currentPosition);    // werror("&&& %O\n", arg);    Type t = parser->parseOrType();    if (!t)
222:      static string valueArgHandler(string keyword, string arg) {    // werror("This is the @type arg handler "); -  .PikeParser parser = .PikeParser(arg); +  .PikeParser parser = .PikeParser(arg, currentPosition);    // werror("&&& %O\n", arg);    string s = parser->parseLiteral() || parser->parseIdents();    if (!s)
295:    return (argHandlers[keyword] || standardArgHandler)(keyword, arg);   }    - static int getKeywordType(string keyword) { -  if (keywordtype[keyword]) -  return keywordtype[keyword]; -  if (strlen(keyword) > 3 && keyword[0..2] == "end") -  return ENDKEYWORD; -  return ERRORKEYWORD; - } -  - static int getTokenType(array(string) | string token) { -  if (arrayp(token)) -  return getKeywordType(token[0]); -  if (!token) -  return ENDTOKEN; -  return TEXTTOKEN; - } -  - static int isSpaceChar(int char) { -  return (< '\t', '\n', ' ' >) [char]; - } -  - static int isKeywordChar(int char) { -  return char >= 'a' && char <= 'z'; - } -  - static array(string) extractKeyword(string line) { -  line += "\0"; -  int i = 0; -  while (i < strlen(line) && isSpaceChar(line[i])) -  ++i; -  if (line[i++] != '@') -  return 0; -  int keywordstart = i; -  while (isKeywordChar(line[i])) -  ++i; -  if (i == keywordstart || line[i] && !isSpaceChar(line[i])) -  return 0; -  string keyword = line[keywordstart .. i - 1]; -  // if (getKeywordType(keyword) == METAKEYWORD) -  return ({ keyword, line[i .. strlen(line) - 2] }); // skippa "\0" ... - } -  - static int allSpaces(string s) { -  for (int i = strlen(s) - 1; i >= 0; --i) -  if (s[i] != ' ' && s[i] != '\t') -  return 0; -  return 1; - } -  - static array(string|array(string)) split(string s) { -  array(string) lines = (s - "\r" - "@\n") / "\n"; -  array(string|array(string)) res = ({ }); -  string currentText = ""; -  foreach(lines, string line) { -  array(string) keyword = extractKeyword(line); -  if (keyword) { -  if (strlen(currentText)) -  res += ({ currentText }); -  currentText = ""; -  res += ({ keyword }); -  } -  else { -  if (allSpaces(line)) -  currentText += "\n"; -  else -  currentText += line + "\n"; -  } -  } -  if (strlen(currentText)) -  res += ({ currentText }); -  return res; - } -  +    static string xmlNode(string s) { /* now, @xml works like @i & @tt */    s += "\0";    string res = "";
457:    return "<p>" + res + "</p>\n";   }    - static class DocParserClass { -  static array(array(string)|string) tokens; -  +     // Read until the next delimiter token on the same level, or to    // the end.    static string xmlText() {    string res = "";    for (;;) { -  switch (getTokenType(tokens[0])) { +  Token token = peekToken(); +  switch (token->type) {    case TEXTTOKEN: -  res += xmlNode(tokens[0]); -  tokens = tokens[1..]; +  res += xmlNode(token->text); +  readToken();    break;    case CONTAINERKEYWORD: -  string keyword = tokens[0][0]; -  string arg = tokens[0][1]; +  string keyword = token->keyword; +  string arg = token->arg;       res += "<" + keyword;    string|mapping(string:string) args = getArgs(keyword, arg);
483:    if (stringp(args))    res += args;    -  tokens = tokens[1..]; +  readToken();    res += xmlContainerContents(keyword); -  if (!(arrayp(tokens[0]) && tokens[0][0] == "end" + keyword)) +  if (!(peekToken()->type == ENDKEYWORD +  && peekToken()->keyword == "end" + keyword))    parseError(sprintf("@%s without matching @end%s",    keyword, keyword));    res += closetag(keyword); -  -  tokens = tokens[1..]; +  readToken();    break;    default:    return res;
500:       static string xmlContainerContents(string container) {    string res = ""; -  switch( getTokenType(tokens[0]) ) { +  Token token = peekToken(); +  switch(token->type) {    case ENDTOKEN:    return "";    case TEXTTOKEN:    // SHOULD WE KILL EMPTY TEXT LIKE THIS ??    { -  string text = tokens[0]; +  string text = token->text;    if (text - "\n" - "\t" - " " == "") { -  tokens = tokens[1..]; +  readToken();    break;    }    else
519:    break;    case ERRORKEYWORD:    //werror("bosse larsson: %O\n", tokens); -  parseError("unknown keyword: @" + tokens[0][0]); +  parseError("unknown keyword: @" + token->keyword);    }    for (;;) { -  if (! (<SINGLEKEYWORD, DELIMITERKEYWORD>) [getTokenType(tokens[0])] ) +  token = peekToken(); +  if (! (<SINGLEKEYWORD, DELIMITERKEYWORD>) [token->type] )    return res;       string single = 0;    array(string) keywords = ({});    res += opentag("group"); -  while ( (<SINGLEKEYWORD, DELIMITERKEYWORD>) [getTokenType(tokens[0])] ) { -  string keyword = tokens[0][0]; -  single = single || getTokenType(tokens[0]) == SINGLEKEYWORD && keyword; +  while ( (<SINGLEKEYWORD, DELIMITERKEYWORD>) [token->type] ) { +  string keyword = token->keyword; +  single = single || (token->type == SINGLEKEYWORD && keyword);    multiset(string) allow = allowedChildren[container];    if (!allow || !allow[keyword])    parseError("@" + keyword + " is not allowed inside @" + container);
541:    parseError("@" + keyword + " may not be grouped together with @" + k);    keywords += ({ keyword });    -  string arg = tokens[0][1]; +  string arg = token->arg;    res += "<" + keyword;    string|mapping(string:string) args = getArgs(keyword, arg);    if (mappingp(args)) {
554:    else    res += "/>";    -  tokens = tokens[1..]; +  readToken(); +  token = peekToken();    } -  switch(getTokenType(tokens[0])) { +  switch(token->type) {    case TEXTTOKEN:    // SHOULD WE KILL EMPTY TEXT LIKE THIS ??    { -  string text = tokens[0]; +  string text = token->text;    if (text - "\n" - "\t" - " " == "") { -  tokens = tokens[1..]; +  readToken();    break;    }    else
577:    }    }    -  static void create(string | array(string|array(string)) s) { -  tokens = (arrayp(s) ? s : split(s)) + ({ 0 }); // end sentinel +  static void create(string | array(Token) s, +  SourcePosition|void position) +  { +  if (arrayp(s)) { +  tokenArr = s;    } -  +  else { +  if (!position) throw(({ __FILE__,__LINE__,"position == 0"})); +  tokenArr = split(s, position); +  } +  tokenArr += ({ Token(ENDTOKEN, 0, 0, 0, 0) }); +  }       MetaData getMetaData() { -  int i = 0; -  while (arrayp(tokens[i]) && getKeywordType(tokens[i][0]) == METAKEYWORD) -  ++i; -  tokens[0 .. i - 1]; +     MetaData meta = MetaData();    string scopeModule = 0; -  foreach (tokens[0 .. i - 1], [string keyword, string arg]) +  while(peekToken()->type == METAKEYWORD) { +  Token token = readToken(); +  string keyword = token->keyword; +  string arg = token->arg; +  string endkeyword = 0;    switch (keyword) {    case "class":    case "module":    { -  +  if (endkeyword) +  parseError("@%s must stand alone", endkeyword);    if (meta->appears)    parseError("@appears before @%s", keyword);    if (meta->belongs)
600:    if (meta->type)    parseError("@%s can not be combined with @%s", keyword, meta->type);    meta->type = keyword; -  .PikeParser nameparser = .PikeParser(arg); +  .PikeParser nameparser = .PikeParser(arg, currentPosition);    string s = nameparser->readToken();    if (!isIdent(s) && s != "::")    parseError("@%s: expected %s name, got %O", keyword, keyword, s);
617:       case "decl":    { +  if (endkeyword) +  parseError("@%s must stand alone", endkeyword);    int first = !meta->type;    if (!meta->type)    meta->type = "decl";
627:    if (meta->belongs)    parseError("@belongs before @decl");    meta->type = "decl"; -  -  .PikeParser declparser = .PikeParser(arg); +  .PikeParser declparser = .PikeParser(arg, currentPosition);    PikeObject p = declparser->parseDecl(    ([ "allowArgListLiterals" : 1,    "allowScopePrefix" : 1 ]) ); // constants/literals + scope::
650:    break;       case "appears": +  if (endkeyword) +  parseError("@%s must stand alone", endkeyword);    if (meta->type == "class" || meta->type == "decl"    || meta->type == "module" || !meta->type)    {
659:    parseError("both @appears and @belongs");    if (scopeModule)    parseError("both 'scope::' and @appears"); -  .PikeParser idparser = .PikeParser(arg); +  .PikeParser idparser = .PikeParser(arg, currentPosition);    string s = idparser->parseIdents();    if (!s)    parseError("@appears: expected identifier, got %O", arg);
670:    break;       case "belongs": +  if (endkeyword) +  parseError("@%s must stand alone", endkeyword);    if (meta->type == "class" || meta->type == "decl"    || meta->type == "module" || !meta->type)    {
679:    parseError("both @appears and @belongs");    if (scopeModule)    parseError("both 'scope::' and @belongs"); -  .PikeParser idparser = .PikeParser(arg); +  .PikeParser idparser = .PikeParser(arg, currentPosition);    string s = idparser->parseIdents();    if (!s && idparser->peekToken() != EOF)    parseError("@belongs: expected identifier or blank, got %O", arg);
690:    case "endclass":    case "endmodule":    { -  if (i > 1) -  parseError("@%s must stand alone", keyword); -  meta->type = keyword; -  .PikeParser nameparser = .PikeParser(arg); +  if (meta->type || meta->belongs || meta->appears) +  parseError("@%s must stand alone", endkeyword); +  meta->type = endkeyword = keyword; +  .PikeParser nameparser = .PikeParser(arg, currentPosition);    while (nameparser->peekToken() != EOF)    meta->name = (meta->name || "") + nameparser->readToken();    }    break; -  +     default:    parseError("illegal keyword: @%s", keyword);    } -  tokens = tokens[i ..]; +  }    if (scopeModule)    meta->belongs = scopeModule;    return meta;    }    -  array(array(string)) getMetaDataOLD() { // OLD VERSION -  // collect all meta info at the start of the block -  int i = 0; -  // werror("\n%%%%%% %O\n", tokens); -  while (arrayp(tokens[i]) && getKeywordType(tokens[i][0]) == METAKEYWORD) -  ++i; -  array(array(string)) metadata = tokens[0..i-1]; -  tokens = tokens[i..]; -  return metadata; -  } -  +     string getDoc(string context) {    string xml = xmlContainerContents(context); -  switch (getTokenType(tokens[0])) { +  switch (peekToken()->type) {    case ENDTOKEN:    break;    case ERRORKEYWORD: -  parseError("illegal keyword: @"+ tokens[0][0]); +  parseError("illegal keyword: @"+ peekToken()->keyword);    default:    parseError("expected end of doc comment");    }
735:      // Each of the arrays in the returned array can be fed to   // Parse::create() - array(array(string|array(string))) splitDocBlock(string block) { -  array(string|array(string)) tokens = split(block); -  array(string|array(string)) current = ({ }); -  array(array(string|array(string))) result = ({ }); + array(array(Token)) splitDocBlock(string block, SourcePosition position) { +  array(Token) tokens = split(block, position); +  array(Token) current = ({ }); +  array(array(Token)) result = ({ });    int prevMeta = 0; -  foreach (tokens, string|array(string) token) { -  int meta = arrayp(token) && getKeywordType(token[0]) == METAKEYWORD; +  foreach (tokens, Token token) { +  int meta = token->type == METAKEYWORD;    if (meta && !prevMeta && sizeof(current)) {    result += ({ current });    current = ({ });
759:   class Parse {    inherit DocParserClass;    static int state; -  static MetaData mMetadata = 0; +  static MetaData mMetaData = 0;    static string mDoc = 0;    static string mContext = 0; -  SourcePosition sourcePos = 0; -  void create(string | array(string|array(string)) s, SourcePosition|void sp) { -  ::create(s); +  void create(string | array(Token) s, SourcePosition|void sp) { +  ::create(s, sp);    state = 0; -  sourcePos = sp; +     }       MetaData metadata() { -  mixed err = catch { +     if (state == 0) {    ++state; -  mMetadata = ::getMetaData(); +  mMetaData = ::getMetaData();    } -  return mMetadata; -  }; -  if (sourcePos) -  throw(({ err[0], sourcePos })); +  return mMetaData;    }       string doc(string context) { -  mixed err = catch { +     if (state == 1) {    ++state;    mContext == context;
791:    else if (state == 0 || state > 1 && mContext != context)    return 0;    return mDoc; -  }; -  if (sourcePos) -  throw(({ err[0], sourcePos })); +     }   }