Diff of pike.git/bin/extract.pike (74c57a9363cee62ff7ef4622714700d778da19dc » e6a74991ef6c10c23431ffa2269d72908b3e9f61)

pike.git/bin/extract.pike:1:   #!/usr/local/bin/pike   // Copyright © 2000, Roxen IS.   // By Martin Nilsson and Andreas Lange   // - // $Id: extract.pike,v 1.2 2000/07/10 17:47:08 lange Exp $ + // $Id: extract.pike,v 1.3 2000/07/14 11:49:04 lange Exp $   //         // The arguments given to the program   mapping args=([]);   // All the files to gather strings from   array(string) files=({});   // All ids used, id:text   mapping(string:string) ids=([]);   // Reversed id mapping, text:id
pike.git/bin/extract.pike:112:    if(instr) ret+="\\";       default:    if(instr) ret+=in[i..i];    }    }    return ret;   }       + function get_encoder(string encoding) { + // If needed, returns a function which encodes a string + if(!encoding || encoding=="") + return 0; + switch(lower_case(encoding)) + { + case "iso-8859-1": + // The normal, no decode needed + return 0; + + case "utf-8": case "utf8": + return lambda(string s) { + return string_to_utf8(s); + }; + + case "utf-16": case "utf16": + case "unicode": + return lambda(string s) { + return string_to_unicode(s); + }; + + default: + object enc; + if(catch(enc = Locale.Charset.encoder( encoding ))) { + werror("\n* Error: Unknown encoding %O!\n", encoding); + exit(1); + } + return lambda(string s) { + return enc->clear()->feed(s)->drain(); + }; + } + } + +   function get_decoder(string encoding) {    // If needed, returns a function which decodes a string    if(!encoding || encoding=="")    return 0;    switch(lower_case(encoding))    {    case "iso-8859-1":    // The normal, no decode needed    return 0;
pike.git/bin/extract.pike:136:       case "utf-16": case "utf16":    case "unicode":    return lambda(string s) {    return unicode_to_string(s);    };       default:    object dec;    if(catch(dec = Locale.Charset.decoder( encoding ))) { - werror("\n* Unknown encoding %O!\n", encoding); + werror("\n* Error: Unknown encoding %O!\n", encoding);    exit(1);    }    return lambda(string s) {    return dec->clear()->feed(s)->drain();    };    }   }         string parse_xml_file(string filename, void|mixed wipe_pass) {    // Reads a language-xml (like project_eng.xml)    // Marks used ids in ids([]), also adds r_ids([text]) from id-comment    // Returns file, with markers instead of <--! [id] ""-->\n<t></t>    // write_xml_file uses the returned data and id_xml_order to build a new one    // Set parameter wipe_pass=1 to remove ids not in ids[] from file    if(!filename || filename=="")    return ""; - Stdio.File in=Stdio.File(); + Stdio.File in=Stdio.FILE();    if(!in->open(filename, "r"))    return ""; - write("Opening %s", filename); + write("Reading %s", filename); + string line = in->gets();    string indata = in->read();    in->close(); - if(!sizeof(indata)) { + if(!indata) {    write("\n");    return "";    } - + + // Check encoding + if(!line) + line = indata; + sscanf(line, "%*sencoding=\"%s\"", string encoding); + if(encoding && encoding!="") { + if(!args->encoding) + // Keep encoding if not overrideed + args->encoding = encoding; + function decode = get_decoder(encoding); + if(decode && catch( indata = decode(indata) )) { + werror("\n* Error: unable to decode from %O in %O\n", + encoding, filename); + exit(1); + } + } + else if(line!=indata) + indata += line+"\n"+indata; +    if(wipe_pass)    write(" - doing wipe pass...");    else    write(" - parsing xml...");       // Comment id mapping - text from , id:text    // text inserted into ids[id] in the t_tag function    mapping c_ids=([]);       Parser.HTML xml_parser = Parser.HTML(); - function decode=0; +    function t_tag =    lambda(object foo, mapping m, string c) {    if(!m->id||m->id=="") {    werror("\n* Warning: String %O has no id.",c);    return 0;    }    if(wipe_pass) {    // This pass is done to remove id's not used anymore    if(!ids[m->id]) {    id_xml_order -= ({ m->id });
pike.git/bin/extract.pike:233:    lambda(object foo, mapping m, string c) {    c = String.trim_whites(c);    if(args->project && args->project!=c) {    werror("\n* xml data is for project %O, not %O!\n",    c,args->project);    exit(1);    } else    args->project=c;    return "\b";    }); - xml_parser-> - add_quote_tag("?xml", - // Check encoding - lambda(object foo, string c) { - sscanf(c,"%*sencoding=\"%s\"", string encoding); - if(encoding && encoding!="") { - if(!args->encoding) - // Keep encoding if not overrideed - args->encoding = encoding; - decode = get_decoder(encoding); - } - return "\b"; - }, "?"); +    xml_parser->add_tag("added",    // Make sure <add>-tags don't get added more than once    lambda(object foo, mapping m) {    m_delete(add,m->id);    added[m->id]=1;    return "\b";    });    xml_parser->    add_quote_tag("!--",    // Might be a normal comment or a 
pike.git/bin/extract.pike:269:    if(id == 0) {    return 0; // Normal comment tag    }    // Really make sure quotings are right    object RE = Regexp("^[^\"]*\"(.*)\"[^\"]*$");    array hits = RE->split(c);    if(hits)    c = get_first_string(sprintf("%O",hits[0]));    // Replace encoded entities    c = replace(c,({"<",">","&"}),({"<",">","&"})); - if(decode) - if(catch( c = decode(c) )) { - werror("\n* Warning: Decoding from %s failed for "+ - "comment with id %s\n", args->encoding,id); - return "\b"; - } +    if(id!="" && c!="")    // Save text for use in the t_tag function    c_ids[id]=c;    return "\b";    }, "--");    // These tags will always be rewritten anyway, so remove them. - + xml_parser->add_quote_tag("?xml", "\b", "?");    xml_parser->add_containers( (["file" : "\b",    "dumped" : "\b",    "language" : "\b"]) );    xml_parser->feed(indata)->finish();       // Remove markers and lines from removed tags    string ret="";    object RE = Regexp("^[\b \t\n]+$");    foreach(xml_parser->read()/"\n", string line) {    if(!RE->match(line))
pike.git/bin/extract.pike:318:    // No ids changed or read with parse_xml_file()    return;    Stdio.File out=Stdio.File();    if(!out->open(out_name, "cw")) {    werror("* Error: Could not open %s for writing\n", out_name);    exit(1);    }       write("Writing %s...",out_name);    - // Default nilencoding - function encode=0; - object _enc; - if(args->encoding) { - // Set encoder function if encoding known. - switch(lower_case(args->encoding)) - { - default: - if(catch(_enc = Locale.Charset.encoder( args->encoding ))) { - werror("\n* Unknown encoding %O, using default", args->encoding); - args->encoding=0; - break; - } - encode = lambda(string s) { - return _enc->clear()->feed(s)->drain(); - }; - break; + // Dump some headers + string newfile=""; + newfile += "<locale version=\"1.0\">\n"; + newfile += "<project>"+args->project+"</project>\n"; + newfile += "<language>English</language>\n"; + newfile += "<dumped>"+time()+"</dumped>\n";    - case "utf-8": case "utf8": - encode = lambda(string s) { - return string_to_utf8(s); - }; - break; - - case "utf-16": case "utf16": - case "unicode": - encode = lambda(string s) { - return string_to_unicode(s); - }; - break; - - case "": - args->encoding = 0; - - case "iso-8859-1": - // No encoding needed - } - } - - // Dump headers - out->write("<?xml version=\"1.0\" encoding=\""+ - (args->encoding||"ISO-8859-1")+"\"?>\n"); - out->write("<locale version=\"1.0\">\n"); - out->write("<project>"+args->project+"</project>\n"); - out->write("<language>English</language>\n"); - out->write("<dumped>"+time()+"</dumped>\n"); - +    // List files included in the project    foreach(files, string inname) - out->write("<file>"+inname+"</file>\n"); + newfile += "<file>"+inname+"</file>\n";       // List blocks added from the config    foreach(indices(added)+indices(add), string blockname) - out->write("<added id=\""+blockname+"\"/>\n"); + newfile += "<added id=\""+blockname+"\"/>\n";       string tag="t";    string info="";    if(args->verbose) {    tag="translate";    info="Original: ";    }       // Reuse structure of old xml    int i=0;
pike.git/bin/extract.pike:393:    string marker = "\7\7\7\7"; // Magic Marker from parse_xml_file()    while(int n=search(outdata, marker)) {    if(n<0) break;    if(i==sizeof(id_xml_order)) {    // Shrinking file?    outdata=replace(outdata,marker,"");    continue;    }    string id=id_xml_order[i];    string str=ids[id]; - if(encode) str=encode(str); // Encode and make parser-safe + // Make parser-safe    str = replace(str, ({"<",">","&"}), ({"<",">","&"}));    outdata = (outdata[0..n-1] +    sprintf("\n<%s id=\"%s\"></%s>",    id, info, str, tag, id, tag) +    outdata[n+sizeof(marker)..sizeof(outdata)-1]);    i++;    } - out->write(outdata); + newfile += outdata;    }       // Dump new strings    while(i<sizeof(id_xml_order)) {    string id=id_xml_order[i];    string str=ids[id]; - if(encode) str=encode(str); // Encode and make parser-safe + // Make parser-safe    str = replace(str, ({"<",">","&"}), ({"<",">","&"})); - out->write("\n\n<%s id=\"%s\"></%s>\n", + newfile += sprintf("\n\n<%s id=\"%s\"></%s>\n",    id, info, str, tag, id, tag);    i++;    }       // If any, add missing <add>-blocks from config    foreach(indices(add), string blockname) - out->write("\n"+add[blockname]); + newfile += "\n"+add[blockname];    - write("\n"); - out->write("\n</locale>\n"); + // Close locale tag + newfile += "\n</locale>\n"; + + // Determine encoding + if(!args->encoding || args->encoding=="") { + int width = String.width( newfile ); + if(width==16) + args->encoding = "utf-8"; + else if(width==32) + args->encoding = "utf-16"; + else + args->encoding = "iso-8859-1"; + } + function encode = get_encoder( args->encoding ); + if(encode) + newfile = encode( newfile ); + newfile = "<?xml version=\"1.0\" encoding=\""+args->encoding+"\"?>\n"+newfile; + + out->write( newfile );    out->truncate( out->tell() );    out->close(); - + write("\n"); +   }      array(string) get_tokens(string in, mapping args, string filename) {    // Picks out tokens from <locale-token>-tag in pikesource    // The order between // blocks and /* */ blocks is not important    // for our purposes.    string comments="";    foreach(in/"//", string line) {    sscanf(line, "%s\n", line);    comments+=line+"\n";
pike.git/bin/extract.pike:480:    foreach(filelist, string filename) {    Stdio.File file=Stdio.File();    if(!file->open(filename, "r")) {    werror("* Error: Could not open sourcefile %s.\n", filename);    exit(1);    }    write("Reading %s",filename);    string indata=file->read();    file->close();    - // Find the tokens + // Get locale tokens, tokenize pike file    write(", parsing...");    array tokens=get_tokens(indata, args, filename); - + mixed pdata = Parser.Pike.split(indata); + pdata = Parser.Pike.tokenize(pdata); + pdata = Parser.Pike.hide_whitespaces(pdata);    - // Replace tokens defined in indata with a suitable (unique) pattern - string presplit = "\"\">>>"; - string midsplit = "<\"\"-\"\">"; - string postsplit = "<<<\"\""; - object(Regexp) RE; - array hits; +    array id_pike_order=({}); - foreach(tokens, string token) { - RE = Regexp("^#[ \t]*define[ \t\n]*"+token); - string newdata = ""; - foreach(indata/"\n", string line) { - if(RE->match(line)) - newdata += ("#define " + token + "(X,Y...) "+ - presplit + "X" + midsplit + "Y" + postsplit + "\n"); + string id, fstr, token; + for(int i=0; i<sizeof(pdata); i++) { + //// Search for tokens + foreach(tokens, token) + if(token==pdata[i]) break; // Loop tokens + if(token!=pdata[i]) continue; // Verify token + if(pdata[++i]!="(") continue; // Verify "(" + + //// Get id + id = (string)pdata[++i]; + if(id=="\"\"") + id="";    else - newdata += line + "\n"; - } + id = get_first_string(id);    - // Preparse data to mark the strings - newdata = cpp(newdata); - // Use this regexp to find the strings - RE = Regexp(presplit+".*\"(.*)\".*"+midsplit+" (.*) "+postsplit); - string id, fstr; - foreach(newdata/"\n", string line) { - hits = RE->split(line); - if(hits && sizeof(hits)==2) { - // String found: get id and decode cpp'ed string - id = hits[0]; - fstr = get_first_string(hits[1]); - if (fstr=="") - continue; // No need to store empty strings + //// Get string + string instr=""; + i++; // Skip "," + while(++i<sizeof(pdata) && pdata[i]!=")") + instr += (string)pdata[i]; + if(fstr=="\"\"") { + if(id=="") + continue; // Neither string nor id, skip! + fstr=""; // Empty string with id, need to save id as used + } else + fstr = get_first_string(instr); + + //// Check and store id and string    if(id == "") {    if (r_ids[fstr])    id = r_ids[fstr]; // Re-use old id with identical string    else    id = make_id(); // New string --> Get new id    // New id for string --> file needs update, save info. - id_pike_order += ({ ({id, token, quotemeta(hits[1])}) }); + id_pike_order += ({ ({id, token, quotemeta(instr)}) });    } else {    // Verify old id    if(!id_origin[id]) {    // Remove preread string in r_ids lookup, might be updated    m_delete(r_ids, ids[id]);    } else {    if(ids[id] && ids[id] != fstr) {    werror("\n* Error: inconsistant use of id.\n");    werror(" In file:%{ %s%}\n",id_origin[id]);    werror(" id %O -> string %O\n",id,ids[id]);    werror(" In file: %s\n",filename);    werror(" id %O -> string %O\n",id,fstr);    exit(1);    }    }    if(r_ids[fstr] && r_ids[fstr]!=id && id_origin[r_ids[fstr]])    werror("\n* Warning: %O has id %O in%{ %s%}, id %O in %s",    fstr, r_ids[fstr], id_origin[r_ids[fstr]], id, filename);    } - if(!has_value(id_xml_order,id)) + if(!has_value(id_xml_order,id) && fstr!="")    // Id not in xml-structure, add to list    id_xml_order += ({id}); - id_origin[id] += ({filename}); // Remember origin - ids[id] = fstr; // Store id:text - r_ids[fstr] = id; // Store text:id + id_origin[id] += ({filename}); // Remember origin + ids[id] = fstr; // Store id:text + if(fstr!="") r_ids[fstr] = id; // Store text:id    } - } - } +    - // Rebuild sourcefile if needed + // Done parsing, rebuild sourcefile if needed    if(!sizeof(id_pike_order)) {    write("\n");    continue;    }    if(!args->nocopy)    filename+=".new"; // Create new file instead of overwriting    write("\n-> Writing %s with new ids: %d",filename,sizeof(id_pike_order));    if(!file->open(filename, "cw")) {    werror("\n* Error: Could not open %s for writing\n", filename);    exit(1);    }       foreach(id_pike_order, array id) {    // Insert ids based on tokens and the now regexp-safe string - + object(Regexp) RE;    // RE = ^(.*TOKEN$ ")(", string $.*)$    RE = Regexp("^(.*" + id[1] + "\$[ \n\t]*\")" +    "(\"[ ,\n\t]*"+id[2]+"[ \t\n]*\$.*)$"); - hits = RE->split(indata); + array hits = RE->split(indata);    if(hits)    indata = hits[0] + id[0] + hits[1];    else    werror("\n* Failed to set id %O for string %O in %s",    id[0], ids[id[0]], filename);    }    write("\n");       file->write(indata);    file->truncate( file->tell() );    file->close();    }   }         void update_xml_sourcefiles(array filelist) {    // Extracts strings from html/xml files in filelist    // Updates ids, r_ids, id_xml_order with ids and strings    // If new ids, updates the sourcefile or a copy    foreach(filelist, string filename) { - Stdio.File file=Stdio.File(); + Stdio.File file=Stdio.FILE();    if(!file->open(filename, "r")) {    werror("* Error: Could not open sourcefile %s.\n", filename);    exit(1);    }    write("Reading %s",filename); - string indata=file->read(); + string line = file->gets(); + string data = file->read();    file->close(); - + if(!data) + continue;    - + // Check encoding + if(!line) + line = data; + string encoding; + sscanf(line, "%*sencoding=\"%s\"", encoding); + if(encoding && encoding!="") { + function decode = get_decoder(encoding); + if(decode && catch( data = decode(data) )) { + werror("\n* Error: unable to decode from %O in %O\n", + encoding, filename); + exit(1); + } + } + else if(line!=data) + data = line+"\n"+data; +    write(", parsing...");    int new = 0;    int ignoretag=0; - function decode=0; +    Parser.HTML xml_parser = Parser.HTML();    xml_parser->case_insensitive_tag(1);    xml_parser-> - add_quote_tag("?xml", - // Check for encoding in <?xml?> - lambda(object foo, string c) { - sscanf(c,"%*sencoding=\"%s\"",string encoding); - if(encoding && encoding!="") { - decode = get_decoder(encoding); - } - return 0; - }, "?"); - xml_parser-> +    add_tag("trans-reg",    // Check the registertag for the right project    lambda(object foo, mapping m) {    if(!m->project || m->project=="") {    werror("\n * Error: Missing project in %s\n",    m->project, filename);    exit(1);    }    if(args->project && m->project!=args->project)    ignoretag=1; // Warning, tags might be from another project
pike.git/bin/extract.pike:645:    lambda(object foo, mapping m, string c) {    if(m->project && m->project!="") {    if(m->project!=args->project)    return 0; // Tag belongs to another project    // else Correct project, proceed    } else    if(ignoretag) // No proj specified, check ignoretag    return 0;    string id = m->id||"";    string fstr = c; - if(decode) - fstr=decode(fstr); +    int updated = 0;    if (fstr=="")    return 0; // No need to store empty strings    if(id == "") {    if (r_ids[fstr])    id = r_ids[fstr]; // Re-use old id with same string    else    id = make_id(); // New string --> Get new id    // Mark that we have a new id here    updated = ++new;
pike.git/bin/extract.pike:693:    r_ids[fstr] = id; // Store text:id    if(updated) {    string ret="<translate id=\""+id+"\"";    foreach(indices(m)-({"id"}), string param)    ret+=" "+param+"=\""+m[param]+"\"";    return ({ ret+">"+c+"</translate>" });    }    // Not updated, do not change    return 0;    }); - xml_parser->feed(indata)->finish(); + xml_parser->feed(data)->finish();       // Rebuild sourcefile if needed    if(!new) {    write("\n");    continue;    } - + data = xml_parser->read(); + if(encoding && encoding!="") { + function encode = get_encoder(encoding); + if(encode && catch( data = encode(data) )) { + werror("\n* Error: unable to encode data in %O\n", encoding); + exit(1); + } + } +    if(!args->nocopy)    filename+=".new"; // Create new file instead of overwriting    write("\n-> Writing %s with new ids: %d", filename, new);    if(!file->open(filename, "cw")) {    werror("\n* Error: Could not open %s for writing\n", filename);    exit(1);    } - write("\n"); +    - file->write( xml_parser->read() ); + file->write( data );    file->truncate( file->tell() );    file->close(); - + write("\n");    }   }         string parse_config(string filename) {    // Read config in xml-format and update args([]) and files({})    // Commandline arguments have precedence    // Returns name of outfile (ie project_eng.xml)    if(!filename || filename=="")    return ""; - Stdio.File in=Stdio.File(); + Stdio.File in=Stdio.FILE();    if(!in->open(filename, "r"))    return ""; - + string line = in->gets();    string indata = in->read();    in->close(); - if(!sizeof(indata)) + if(!indata)    return "";    - + // Check encoding + if(!line) + line = indata; + sscanf(line, "%*sencoding=\"%s\"", string encoding); + if(encoding && encoding!="") { + function decode = get_decoder(encoding); + if(decode && catch( indata = decode(indata) )) { + werror("\n* Error: unable to decode from %O in %O\n", + encoding, filename); + exit(1); + } + } + else if(line!=indata) + indata = line+"\n"+indata; +    string xml_name=""; - function decode=0; +    Parser.HTML xml_parser = Parser.HTML();    xml_parser->case_insensitive_tag(1);    xml_parser-> - add_quote_tag("?xml", - // Check for encoding in <?xml?> - lambda(object foo, string c) { - sscanf(c,"%*sencoding=\"%s\"",string encoding); - if(encoding && encoding!="") { - decode = get_decoder(encoding); - } - return 0; - }, "?"); - xml_parser-> +    add_container("project",    // Only read config for the right project, or the    // first found if unspecified    lambda(object foo, mapping m, string c) {    if(!m->name || m->name=="") {    werror("\n* Projectname missing in %s!\n", filename);    exit(1);    } - if( decode && catch(c = decode(c)) ) { - werror("\n* Failed to decode %O in config\n",m->name); - exit(1); - } +    if(args->project && args->project!="" &&    args->project!=m->name)    return ""; // Skip this project-tag    else    args->project = m->name;    write("Reading config for project %O in %s\n",    args->project, filename);    return c;    });    xml_parser->    add_container("out",    // Set outname (default: project_eng.xml)    lambda(object foo, mapping m, string c) { - if( decode && catch(c = decode(c)) ) { - werror("\n* Failed to decode %O in config\n",m->name); - exit(1); - } +    c = String.trim_whites(c);    if(c && c!="")    xml_name = c;    return 0;    });    xml_parser->    add_container("file",    // Add a file to be parsed    lambda(object foo, mapping m, string c) { - if( decode && catch(c = decode(c)) ) { - werror("\n* Failed to decode %O in config\n",m->name); - exit(1); - } +    c = String.trim_whites(c);    if(c && c!="")    files += ({ c });    return 0;    });    xml_parser->    add_container("encoding",    // Set default encoding    lambda(object foo, mapping m, string c) { - if( decode && catch(c = decode(c)) ) { - werror("\n* Failed to decode %O in config\n",m->name); - exit(1); - } + if(args->encoding=="") + args->encoding = 0;    c = String.trim_whites(c); - if(c && c!="" && !args->encoding) + if(c && c!="" && !args->encoding) {    args->encoding = c; - + get_encoder( c ); // Check if known + }    return 0;    });    xml_parser->    add_container("includepath",    // Add includepath needed for pikefiles    lambda(object foo, mapping m, string c) { - if( decode && catch(c = decode(c)) ) { - werror("\n* Failed to decode %O in config\n",m->name); - exit(1); - } - c = String.trim_whites(c); +    if(c && c!="")    add_include_path(c);    return 0;    });    xml_parser->    add_container("add",    // Block to add to project-xml-files    lambda(object foo, mapping m, string c) {    if(!m->id || m->id=="") {    werror("\n* Missing id in <add> in %s!\n", filename);    exit(1);    } - if( decode && catch(c = decode(c)) ) { - werror("\n* Failed to decode %O in config\n",m->name); - exit(1); - } +    add[m->id]=c;    return 0;    });    xml_parser->add_tag("nocopy",    // Update the infile instead of creating infile.new    lambda(object foo, mapping m) {    args->nocopy=1;    return 0;    });    xml_parser->add_tag("verbose",
pike.git/bin/extract.pike:891:       // Read configfile    string configname = args->config;    if(!configname && args->project)    configname = args->project+".xml";    string filename = parse_config(configname);    if(filename!="" && (!xml_name || xml_name==""))    xml_name = filename;       if(!sizeof(files) || args->help) { - sscanf("$Revision: 1.2 $", "$"+"Revision: %s $", string v); + sscanf("$Revision: 1.3 $", "$"+"Revision: %s $", string v);    werror("\n Locale Extractor Utility "+v+"\n\n");    werror(" Syntax: extract.pike [arguments] infile(s)\n\n");    werror(" Arguments: --project=name default: first found in infile\n");    werror(" --config=file default: [project].xml\n");    werror(" --out=file default: [project]_eng.xml\n");    werror(" --nocopy update infile instead of infile.new\n");    werror(" --wipe remove unused ids from xml\n");    werror(" --encoding=enc default: ISO-8859-1\n");    werror(" --verbose more informative text in xml\n");    werror("\n");

pike.git / bin / extract.pike