7c006e2000-08-04Andreas Lange #!/usr/local/bin/pike
57fb082000-07-09Martin Nilsson // By Martin Nilsson and Andreas Lange //
392f962002-01-16Martin Nilsson // $Id: extract.pike,v 1.12 2002/01/16 03:12:32 nilsson Exp $
57fb082000-07-09Martin Nilsson // // The arguments given to the program
056e052000-07-17Andreas Lange mapping args = ([]);
57fb082000-07-09Martin Nilsson // All the files to gather strings from
056e052000-07-17Andreas Lange array(string) files = ({});
7c006e2000-08-04Andreas Lange // All ids used, id:mapping(info) mapping ids = ([]);
57fb082000-07-09Martin Nilsson // Reversed id mapping, text:id
7c006e2000-08-04Andreas Lange mapping(string:string|int) r_ids = ([]); // Order of the ids in the xml outdata file array(string|int) id_xml_order = ({}); // Code to add to xml outfile, id:code
056e052000-07-17Andreas Lange mapping(string:string) add = ([]);
7c006e2000-08-04Andreas Lange // List of ids already in the xml outfile
056e052000-07-17Andreas Lange multiset(string) added = (<>);
7c006e2000-08-04Andreas Lange // The highest int with all lower ids set; see make_id()
056e052000-07-17Andreas Lange int high_int_id = 0;
57fb082000-07-09Martin Nilsson 
7c006e2000-08-04Andreas Lange int make_id() {
57fb082000-07-09Martin Nilsson  // Returns the next unused unique id
7c006e2000-08-04Andreas Lange  while ( has_value(id_xml_order, ++high_int_id) ); return high_int_id;
57fb082000-07-09Martin Nilsson } string get_first_string(string in) { // Merges parts, compiles and returns the first string in a line from cpp // ie '"a\\n" "b: " "%s!", string' --> "a\nb: %s!"
056e052000-07-17Andreas Lange  string ret = ""; int instr = 0; for(int i = 0; i<sizeof(in); i++) {
57fb082000-07-09Martin Nilsson  if(in[i]=='\"') if(!(i>0 && in[i-1]=='\\')) { instr= instr? 0 : 1; if(instr) i++; } if(instr) ret+=in[i..i]; else if(in[i]==',') break; } return compile_string("constant q=#\""+ret+"\";")->q; } string quotemeta(string in) { // Takes a string from cpp and quotes it so it will be // regexp-safe and match the string in the source-file
056e052000-07-17Andreas Lange  string ret = ""; int instr = 0; for(int i = 0; i<sizeof(in); i++) {
57fb082000-07-09Martin Nilsson  switch (in[i]) { case '\"': if(!(i>0 && in[i-1]=='\\')) { instr = instr? 0 : 1; if(instr && i>0)
056e052000-07-17Andreas Lange  ret += ".*";
57fb082000-07-09Martin Nilsson  }
056e052000-07-17Andreas Lange  ret += "\"";
57fb082000-07-09Martin Nilsson  break; case '\\': if((i+1)<sizeof(in) && in[i+1]=='n') { if(instr) {
056e052000-07-17Andreas Lange  ret += "[\n|\\\\]n*"; // Must handle both "\\n" and '\n'
57fb082000-07-09Martin Nilsson  i++; } break; }
056e052000-07-17Andreas Lange  case '.': case '+': case '*': case '^': case '(': case ')': case '$': case '[': case ']': case '|': if(instr) ret += "\\";
57fb082000-07-09Martin Nilsson  default:
056e052000-07-17Andreas Lange  if(instr) ret += in[i..i];
57fb082000-07-09Martin Nilsson  } } return ret; }
e6a7492000-07-14Andreas Lange function get_encoder(string encoding) { // If needed, returns a function which encodes a string if(!encoding || encoding=="") return 0;
056e052000-07-17Andreas Lange  switch( lower_case(encoding) )
e6a7492000-07-14Andreas Lange  { case "utf-8": case "utf8": return lambda(string s) { return string_to_utf8(s); }; case "utf-16": case "utf16": case "unicode": return lambda(string s) { return string_to_unicode(s); }; default: object enc;
056e052000-07-17Andreas Lange  if(catch( enc = Locale.Charset.encoder( encoding ) )) {
e6a7492000-07-14Andreas Lange  werror("\n* Error: Unknown encoding %O!\n", encoding); exit(1); } return lambda(string s) { return enc->clear()->feed(s)->drain(); }; } }
57fb082000-07-09Martin Nilsson function get_decoder(string encoding) { // If needed, returns a function which decodes a string
878f212000-07-10Andreas Lange  if(!encoding || encoding=="") return 0;
056e052000-07-17Andreas Lange  switch( lower_case(encoding) )
57fb082000-07-09Martin Nilsson  {
878f212000-07-10Andreas Lange  case "iso-8859-1": // The normal, no decode needed return 0;
57fb082000-07-09Martin Nilsson  case "utf-8": case "utf8": return lambda(string s) { return utf8_to_string(s); }; case "utf-16": case "utf16": case "unicode": return lambda(string s) { return unicode_to_string(s); };
878f212000-07-10Andreas Lange  default: object dec;
056e052000-07-17Andreas Lange  if(catch( dec = Locale.Charset.decoder( encoding ) )) {
e6a7492000-07-14Andreas Lange  werror("\n* Error: Unknown encoding %O!\n", encoding);
878f212000-07-10Andreas Lange  exit(1); } return lambda(string s) { return dec->clear()->feed(s)->drain(); };
57fb082000-07-09Martin Nilsson  } }
ae23712000-08-11Andreas Lange array(mapping) languagefiles(string searchpath, void|string skiplang) {
7c006e2000-08-04Andreas Lange  // Based on the searchpath, returns list of files - skiplang-file string pattern = replace(searchpath, "%%", "%"); string dirbase = (pattern/"%L")[0];
ca612c2001-02-20Martin Nilsson  if(dirbase=="") { dirbase="./"; pattern = "./" + pattern; } else if(dirbase[-1]!='/') {
7c006e2000-08-04Andreas Lange  array split = dirbase/"/"; dirbase = split[..sizeof(split)-2]*"/"+"/"; }
ca612c2001-02-20Martin Nilsson 
7c006e2000-08-04Andreas Lange  string s_patt; if(search(pattern, "/", sizeof(dirbase))==-1) s_patt=pattern[sizeof(dirbase)..]; else s_patt=pattern[sizeof(dirbase)..search(pattern, "/", sizeof(dirbase))-1]; s_patt = replace(s_patt, "%L", "%3s"); array dirlist = get_dir(dirbase); if(!dirlist) return ({}); array list = ({}); foreach(dirlist, string path) { string lang; if(!sscanf(path, s_patt, lang)) continue; if(lang==skiplang) continue; string file = replace(pattern, "%L", lang); if(!file_stat(file)) continue;
ae23712000-08-11Andreas Lange  list += ({ (["name":file, "lang":lang]) });
7c006e2000-08-04Andreas Lange  } return list; }
ae23712000-08-11Andreas Lange mapping parse_xml_file(string filename, string language) {
57fb082000-07-09Martin Nilsson  // Reads a language-xml (like project_eng.xml)
7c006e2000-08-04Andreas Lange  // Marks used ids in ids([]), also adds r_ids([text])
ae23712000-08-11Andreas Lange  // Returns mapping, // 'encoding' = file encoding, // 'data'= file with markers instead of <str>-blocks
056e052000-07-17Andreas Lange  // write_xml_file uses the returned data+id_xml_order to build a new one
7c006e2000-08-04Andreas Lange  added = (<>); id_xml_order = ({});
57fb082000-07-09Martin Nilsson  if(!filename || filename=="")
ae23712000-08-11Andreas Lange  return ([]);
e6a7492000-07-14Andreas Lange  Stdio.File in=Stdio.FILE();
57fb082000-07-09Martin Nilsson  if(!in->open(filename, "r"))
ae23712000-08-11Andreas Lange  return ([]); write("Reading %s%s", language ? "["+language+"] " : "", (filename/"/")[-1]);
e6a7492000-07-14Andreas Lange  string line = in->gets();
57fb082000-07-09Martin Nilsson  string indata = in->read(); in->close();
e6a7492000-07-14Andreas Lange  if(!indata) {
57fb082000-07-09Martin Nilsson  write("\n");
ae23712000-08-11Andreas Lange  return ([]);
e6a7492000-07-14Andreas Lange  } // Check encoding
ae23712000-08-11Andreas Lange  string encoding;
e6a7492000-07-14Andreas Lange  if(!line) line = indata;
ae23712000-08-11Andreas Lange  sscanf(line, "%*sencoding=\"%s\"", encoding);
e6a7492000-07-14Andreas Lange  if(encoding && encoding!="") { function decode = get_decoder(encoding); if(decode && catch( indata = decode(indata) )) { werror("\n* Error: unable to decode from %O in %O\n", encoding, filename); exit(1); } } else if(line!=indata) indata += line+"\n"+indata;
7c006e2000-08-04Andreas Lange  write(" - parsing xml...");
57fb082000-07-09Martin Nilsson 
7c006e2000-08-04Andreas Lange  // Parse... First the <str>-parser mapping current = ([]); Parser.HTML str_parser = Parser.HTML(); str_parser->case_insensitive_tag(1);
57fb082000-07-09Martin Nilsson 
7c006e2000-08-04Andreas Lange  str_parser-> add_tag("changed", lambda(object foo, mapping m) { current->changetag = str_parser->current()+"\n"; return 0; }); function t_container =
57fb082000-07-09Martin Nilsson  lambda(object foo, mapping m, string c) {
7c006e2000-08-04Andreas Lange  if((int)m->id) m->id = (int)m->id; if(!current->id) { if(!m->id || m->id=="") { werror("\n* Warning: String %O has no id.", c||current->original); return 0; } current->id = m->id; } if(m->id && (m->id != current->id)) { werror("\n* Warning: Ignoring string %O. " "Contained in id %O but marked with id %O.", c, current->id, m->id);
57fb082000-07-09Martin Nilsson  return 0; }
7c006e2000-08-04Andreas Lange  if(has_value(id_xml_order, current->id)) { werror("\n* Error: Id %O used more than once.\n", current->id); exit(1);
57fb082000-07-09Martin Nilsson  }
7c006e2000-08-04Andreas Lange  id_xml_order += ({ current->id }); c = replace(c, ({"&lt;","&gt;","&amp;"}), ({"<",">","&"})); current->text = c; current->textargs = m-({"id"}); return 0;
57fb082000-07-09Martin Nilsson  };
7c006e2000-08-04Andreas Lange  str_parser->add_containers( ([ "t" : t_container, "translate" : t_container ]) );
57fb082000-07-09Martin Nilsson 
7c006e2000-08-04Andreas Lange  function o_container = lambda(object foo, mapping m, string c) { if(String.trim_whites(c)!="") { // Replace encoded entities c = replace(c, ({"&lt;","&gt;","&amp;"}), ({"<",">","&"})); current->original = c; current->originalargs = m-({"id"}); } return 0; }; str_parser->add_containers( ([ "o" : o_container, "original" : o_container ]) ); // Main xml file parser
57fb082000-07-09Martin Nilsson  // "\b" is used as a marker for lines to remove from returned data
7c006e2000-08-04Andreas Lange  Parser.HTML xml_parser = Parser.HTML();
57fb082000-07-09Martin Nilsson  xml_parser->case_insensitive_tag(1);
ae23712000-08-11Andreas Lange  xml_parser->add_quote_tag("!--", lambda() {return 0;}, "--");
57fb082000-07-09Martin Nilsson  xml_parser->
7c006e2000-08-04Andreas Lange  add_container("str",
57fb082000-07-09Martin Nilsson  lambda(object foo, mapping m, string c) {
7c006e2000-08-04Andreas Lange  current = ([]); // New <str>, clear slate if(m->id && m->id!="") { if((int)m->id) m->id = (int)m->id; current->id = m->id; } str_parser->feed( c )->finish(); if(current->id) { ids[current->id] = current; if(String.trim_whites(current->original)!="") r_ids[current->original] = current->id;
57fb082000-07-09Martin Nilsson  }
7c006e2000-08-04Andreas Lange  if(has_value(id_xml_order, current->id)) // Return marker for write_xml_file() // - where to re-insert <str> again. // This is done to make sure the file // really is updated. return "\7\7\7\7"; // Should be unique enough return "\b";
57fb082000-07-09Martin Nilsson  });
7c006e2000-08-04Andreas Lange  xml_parser-> add_tag("locale", // Verify the <locale>-xml version lambda(object foo, mapping m) { array n = m->version/"."; if(n[0]!="1") { werror("\n* Unknown locale version %O!\n", m->version); exit(1); } return "\b"; });
57fb082000-07-09Martin Nilsson  xml_parser-> add_container("project", // Verify that the file is for the this project lambda(object foo, mapping m, string c) { c = String.trim_whites(c); if(args->project && args->project!=c) { werror("\n* xml data is for project %O, not %O!\n",
056e052000-07-17Andreas Lange  c, args->project);
57fb082000-07-09Martin Nilsson  exit(1); } else
056e052000-07-17Andreas Lange  args->project = c;
57fb082000-07-09Martin Nilsson  return "\b"; }); xml_parser->add_tag("added", // Make sure <add>-tags don't get added more than once lambda(object foo, mapping m) {
056e052000-07-17Andreas Lange  m_delete(add, m->id); added[m->id] = 1;
57fb082000-07-09Martin Nilsson  return "\b"; }); // These tags will always be rewritten anyway, so remove them.
e6a7492000-07-14Andreas Lange  xml_parser->add_quote_tag("?xml", "\b", "?");
57fb082000-07-09Martin Nilsson  xml_parser->add_containers( (["file" : "\b", "dumped" : "\b", "language" : "\b"]) ); xml_parser->feed(indata)->finish();
e6a7492000-07-14Andreas Lange 
57fb082000-07-09Martin Nilsson  // Remove markers and lines from removed tags
056e052000-07-17Andreas Lange  string ret = "";
57fb082000-07-09Martin Nilsson  object RE = Regexp("^[\b \t\n]+$"); foreach(xml_parser->read()/"\n", string line) { if(!RE->match(line)) ret += line+"\n"; } // Remove silly lines in end of data RE = Regexp("^(.*[^\n \t]\n)[ \n\t]*$");
7c006e2000-08-04Andreas Lange  array hits = RE->split( ret );
57fb082000-07-09Martin Nilsson  if(hits) ret = hits[0];
7c006e2000-08-04Andreas Lange  ret = replace(ret, "\n\n\n\n", "\n\n"); write("\n");
ae23712000-08-11Andreas Lange  return ([ "encoding":encoding, "data":ret ]);
57fb082000-07-09Martin Nilsson }
ae23712000-08-11Andreas Lange void write_xml_file(string filename, string language, string encoding, string outdata, void|mapping old_ids) // Updates/creates a language xml-file with id:text-info
57fb082000-07-09Martin Nilsson  // Reuses a present structure if fead with it in outdata
ae23712000-08-11Andreas Lange  // Some headers are always rewritten.
7c006e2000-08-04Andreas Lange  // The old_ids mapping is supplied when the file is updated in comparison // with a base xml file. {
57fb082000-07-09Martin Nilsson  if(!sizeof(id_xml_order)) // No ids changed or read with parse_xml_file() return; Stdio.File out=Stdio.File();
ae23712000-08-11Andreas Lange  if(!out->open(filename, "cw")) { werror("* Error: Could not open %s for writing\n", filename);
57fb082000-07-09Martin Nilsson  exit(1); }
ae23712000-08-11Andreas Lange  write("Writing %s%s... (%d ids) ", language ? "["+language+"] " : "", (filename/"/")[-1], sizeof(id_xml_order));
57fb082000-07-09Martin Nilsson 
e6a7492000-07-14Andreas Lange  // Dump some headers
056e052000-07-17Andreas Lange  string newfile = "";
7c006e2000-08-04Andreas Lange  newfile += "<locale version=\"1.0\"/>\n";
e6a7492000-07-14Andreas Lange  newfile += "<project>"+args->project+"</project>\n";
ae23712000-08-11Andreas Lange  newfile += "<language>" + #ifdef constant(Standards.ISO639_2) Standards.ISO639_2.get_language(language) || #endif language + "</language>\n";
6dda652000-11-27Martin Nilsson  if(!args->notime) newfile += "<dumped>"+time()+"</dumped>\n";
57fb082000-07-09Martin Nilsson  // List files included in the project
6dda652000-11-27Martin Nilsson  foreach(sort(files), string inname)
e6a7492000-07-14Andreas Lange  newfile += "<file>"+inname+"</file>\n";
57fb082000-07-09Martin Nilsson  // List blocks added from the config
6dda652000-11-27Martin Nilsson  foreach(sort(indices(added)+indices(add)), string blockname)
e6a7492000-07-14Andreas Lange  newfile += "<added id=\""+blockname+"\"/>\n";
57fb082000-07-09Martin Nilsson 
7c006e2000-08-04Andreas Lange  string o_tag = "o"; string t_tag = "t";
57fb082000-07-09Martin Nilsson  if(args->verbose) {
7c006e2000-08-04Andreas Lange  o_tag = "original"; t_tag = "translate";
57fb082000-07-09Martin Nilsson  }
7c006e2000-08-04Andreas Lange 
ae23712000-08-11Andreas Lange  mapping stats = ([]);
7c006e2000-08-04Andreas Lange  function gen_tag = lambda(mixed id) {
ae23712000-08-11Andreas Lange  stats->written++;
7c006e2000-08-04Andreas Lange  string diff = ((old_ids && old_ids[id] && old_ids[id]->changetag) ? old_ids[id]->changetag : "");
ae23712000-08-11Andreas Lange  if(old_ids) { if(diff!="") stats->changed++; else if(!old_ids[id] || !old_ids[id]->text || String.trim_whites(old_ids[id]->text)=="" ) { diff = "<new/>\n"; stats->new++; }
7c006e2000-08-04Andreas Lange  else if(old_ids[id] && old_ids[id]->original != ids[id]->original) { diff = replace(old_ids[id]->original||"", ({"<",">","&"}), ({"&lt;","&gt;","&amp;"})); diff = "<changed from=\""+ diff +"\"/>\n";
ae23712000-08-11Andreas Lange  stats->changed++;
7c006e2000-08-04Andreas Lange  }
ae23712000-08-11Andreas Lange  else stats->ok++;
7c006e2000-08-04Andreas Lange  } // Make parser-safe string original = replace(ids[id]->original, ({"<",">","&"}), ({"&lt;","&gt;","&amp;"})); string text = replace( ( (old_ids && old_ids[id] && old_ids[id]->text) ? old_ids[id]->text : ""), ({"<",">","&"}),({"&lt;","&gt;","&amp;"})); return sprintf("<str id=\"%s\">\n" "%s<%s>%s</%[2]s>\n" "<%s>%s</%[4]s>\n" "</str>", (string)id, diff, o_tag, original, t_tag, text); };
57fb082000-07-09Martin Nilsson  // Reuse structure of old xml
056e052000-07-17Andreas Lange  int i = 0;
57fb082000-07-09Martin Nilsson  if(outdata) {
7c006e2000-08-04Andreas Lange  string marker = "\7\7\7\7"; // Marker from parse_xml_file() string newstr;
056e052000-07-17Andreas Lange  while( int n=search(outdata, marker) ) {
57fb082000-07-09Martin Nilsson  if(n<0) break; if(i==sizeof(id_xml_order)) { // Shrinking file?
056e052000-07-17Andreas Lange  outdata = replace(outdata, marker, "");
878f212000-07-10Andreas Lange  continue;
57fb082000-07-09Martin Nilsson  }
7c006e2000-08-04Andreas Lange  if(args->wipe && !ids[id_xml_order[i]]->origin) newstr = ""; // Wipe this old string else newstr = gen_tag(id_xml_order[i]); outdata = (outdata[0..n-1] + newstr +
57fb082000-07-09Martin Nilsson  outdata[n+sizeof(marker)..sizeof(outdata)-1]); i++; }
e6a7492000-07-14Andreas Lange  newfile += outdata;
57fb082000-07-09Martin Nilsson  } // Dump new strings
7c006e2000-08-04Andreas Lange  for(; i<sizeof(id_xml_order); i++) { if(!(args->wipe && !ids[id_xml_order[i]]->origin)) newfile += "\n" + gen_tag(id_xml_order[i]) + "\n";
57fb082000-07-09Martin Nilsson  } // If any, add missing <add>-blocks from config foreach(indices(add), string blockname)
e6a7492000-07-14Andreas Lange  newfile += "\n"+add[blockname]; // Determine encoding
ae23712000-08-11Andreas Lange  if(!encoding || encoding=="") {
e6a7492000-07-14Andreas Lange  int width = String.width( newfile ); if(width==16)
ae23712000-08-11Andreas Lange  encoding = "utf-8";
e6a7492000-07-14Andreas Lange  else if(width==32)
ae23712000-08-11Andreas Lange  encoding = "utf-16";
e6a7492000-07-14Andreas Lange  else
ae23712000-08-11Andreas Lange  encoding = "iso-8859-1"; } function encode = get_encoder( encoding ); if(encode && catch( newfile = encode(newfile) )) { werror("\n* Error: unable to encode file %O in %O\n", filename, args->encoding); exit(1);
e6a7492000-07-14Andreas Lange  }
ae23712000-08-11Andreas Lange  newfile = "<?xml version=\"1.0\" encoding=\""+ encoding +"\"?>\n"+ newfile;
57fb082000-07-09Martin Nilsson 
e6a7492000-07-14Andreas Lange  out->write( newfile );
57fb082000-07-09Martin Nilsson  out->truncate( out->tell() ); out->close();
7c006e2000-08-04Andreas Lange 
ae23712000-08-11Andreas Lange  // Dump some statistics if(args->wipe && stats->written!=sizeof(id_xml_order)) write("(wiped to %d) ", stats->written); if(old_ids) { if(stats->written==stats->ok) write("all translated"); else { array ret= ({}); if(stats->ok) ret += ({ sprintf("%d translated", stats->ok) }); if(stats->new) ret += ({ sprintf("%d new", stats->new) }); if(stats->changed) ret += ({ sprintf("%d changed", stats->changed) }); write(String.implode_nicely( ret )); }
7c006e2000-08-04Andreas Lange  } write("\n");
57fb082000-07-09Martin Nilsson }
056e052000-07-17Andreas Lange 
57fb082000-07-09Martin Nilsson array(string) get_tokens(string in, mapping args, string filename) { // Picks out tokens from <locale-token>-tag in pikesource // The order between // blocks and /* */ blocks is not important // for our purposes.
056e052000-07-17Andreas Lange  string comments = "";
58c31c2001-09-27Martin Nilsson  foreach( (in/"//")[1..], string line) {
57fb082000-07-09Martin Nilsson  sscanf(line, "%s\n", line);
056e052000-07-17Andreas Lange  comments += line+"\n";
57fb082000-07-09Martin Nilsson  }
58c31c2001-09-27Martin Nilsson  // This is code is flawed. Breaks in e.g. userfs.pike in Roxen. // foreach(in/"/\052", string block) { // string c = ""; // sscanf(block, "%s\052/", c); // comments += c+"\n"; // }
57fb082000-07-09Martin Nilsson 
056e052000-07-17Andreas Lange  array(string) tokens = ({});
57fb082000-07-09Martin Nilsson  Parser.HTML()-> add_container("locale-token", lambda(object foo, mapping m, string c) {
58c31c2001-09-27Martin Nilsson  if(args->project && m->project!=args->project)
57fb082000-07-09Martin Nilsson  return 0;
056e052000-07-17Andreas Lange  c = String.trim_whites(c);
58c31c2001-09-27Martin Nilsson  if(has_value(tokens, c)) { werror("\n* Warning: Token %O already found.\n", c); }
056e052000-07-17Andreas Lange  tokens += ({c});
57fb082000-07-09Martin Nilsson  if (m->project)
056e052000-07-17Andreas Lange  args->project = m->project;
57fb082000-07-09Martin Nilsson  else
056e052000-07-17Andreas Lange  args->project = "";
57fb082000-07-09Martin Nilsson  return 0; })
7c006e2000-08-04Andreas Lange  ->feed( comments )->finish();
57fb082000-07-09Martin Nilsson  if(!sizeof(tokens)) { if(args->project)
056e052000-07-17Andreas Lange  werror("\n* Warning: No token for project %O in %s\n", args->project, filename);
57fb082000-07-09Martin Nilsson  else
056e052000-07-17Andreas Lange  werror("\n* Warning: No token found in file %s\n", filename);
57fb082000-07-09Martin Nilsson  } return tokens; } void update_pike_sourcefiles(array filelist) { // Extracts strings from pike sourcefiles in filelist
7c006e2000-08-04Andreas Lange  // Updates ids, r_ids and id_xml_order with ids and strings
57fb082000-07-09Martin Nilsson  // If new ids, updates the sourcefile or a copy foreach(filelist, string filename) {
056e052000-07-17Andreas Lange  Stdio.File file = Stdio.File();
57fb082000-07-09Martin Nilsson  if(!file->open(filename, "r")) { werror("* Error: Could not open sourcefile %s.\n", filename); exit(1); }
056e052000-07-17Andreas Lange  write("Reading %s", filename); string indata = file->read();
57fb082000-07-09Martin Nilsson  file->close();
e6a7492000-07-14Andreas Lange  // Get locale tokens, tokenize pike file
57fb082000-07-09Martin Nilsson  write(", parsing...");
056e052000-07-17Andreas Lange  array tokens = get_tokens(indata, args, filename); if(!sizeof(tokens)) continue;
e6a7492000-07-14Andreas Lange  mixed pdata = Parser.Pike.split(indata); pdata = Parser.Pike.tokenize(pdata); pdata = Parser.Pike.hide_whitespaces(pdata);
57fb082000-07-09Martin Nilsson 
056e052000-07-17Andreas Lange  array id_pike_order = ({}); int no_of_ids = 0;
7c006e2000-08-04Andreas Lange  string|int id; string fstr, token;
e6a7492000-07-14Andreas Lange  for(int i=0; i<sizeof(pdata); i++) { //// Search for tokens foreach(tokens, token) if(token==pdata[i]) break; // Loop tokens if(token!=pdata[i]) continue; // Verify token if(pdata[++i]!="(") continue; // Verify "(" //// Get id
7c006e2000-08-04Andreas Lange  id = (string)pdata[++i];
e6a7492000-07-14Andreas Lange  if(id=="\"\"")
056e052000-07-17Andreas Lange  id = "";
7c006e2000-08-04Andreas Lange  else if((int)id) id = (int)id; else
e6a7492000-07-14Andreas Lange  id = get_first_string(id); //// Get string
056e052000-07-17Andreas Lange  string instr = "";
e6a7492000-07-14Andreas Lange  i++; // Skip ","
056e052000-07-17Andreas Lange  while( ++i<sizeof(pdata) && pdata[i]!=")" )
e6a7492000-07-14Andreas Lange  instr += (string)pdata[i];
7c006e2000-08-04Andreas Lange  if(instr=="\"\"") fstr = ""; else
e6a7492000-07-14Andreas Lange  fstr = get_first_string(instr);
7c006e2000-08-04Andreas Lange  if(fstr=="" && id=="") continue; // Neither string nor id, skip!
e6a7492000-07-14Andreas Lange  //// Check and store id and string
056e052000-07-17Andreas Lange  no_of_ids++;
7c006e2000-08-04Andreas Lange  if(!id || id=="") {
e6a7492000-07-14Andreas Lange  if (r_ids[fstr]) id = r_ids[fstr]; // Re-use old id with identical string
57fb082000-07-09Martin Nilsson  else
e6a7492000-07-14Andreas Lange  id = make_id(); // New string --> Get new id // New id for string --> file needs update, save info. id_pike_order += ({ ({id, token, quotemeta(instr)}) }); } else { // Verify old id
7c006e2000-08-04Andreas Lange  if(!ids[id] || (ids[id] && !ids[id]->origin)) {
e6a7492000-07-14Andreas Lange  // Remove preread string in r_ids lookup, might be updated m_delete(r_ids, ids[id]); } else {
7c006e2000-08-04Andreas Lange  if(ids[id] && ids[id]->original!=fstr) {
e6a7492000-07-14Andreas Lange  werror("\n* Error: inconsistant use of id.\n");
7c006e2000-08-04Andreas Lange  werror(" In file:%{ %s%}\n", ids[id]->origin); werror(" id %O -> string %O\n", id, ids[id]->original);
056e052000-07-17Andreas Lange  werror(" In file: %s\n", filename); werror(" id %O -> string %O\n", id, fstr);
e6a7492000-07-14Andreas Lange  exit(1);
57fb082000-07-09Martin Nilsson  } }
7c006e2000-08-04Andreas Lange  if(r_ids[fstr] && r_ids[fstr]!=id && ids[r_ids[fstr]]->origin)
e6a7492000-07-14Andreas Lange  werror("\n* Warning: %O has id %O in%{ %s%}, id %O in %s",
7c006e2000-08-04Andreas Lange  fstr, r_ids[fstr], ids[r_ids[fstr]]->origin, id, filename);
57fb082000-07-09Martin Nilsson  }
7c006e2000-08-04Andreas Lange  if(!has_value(id_xml_order, id))
e6a7492000-07-14Andreas Lange  // Id not in xml-structure, add to list
7c006e2000-08-04Andreas Lange  id_xml_order += ({ id }); if(!ids[id]) ids[id] = ([]); ids[id]->original = fstr; // Store id:text ids[id]->origin += ({filename}); // Add origin if(String.trim_whites(fstr)!="") r_ids[fstr] = id; // Store text:id
57fb082000-07-09Martin Nilsson  }
e6a7492000-07-14Andreas Lange  // Done parsing, rebuild sourcefile if needed
7c006e2000-08-04Andreas Lange  write(" (%d localization%s)\n", no_of_ids, (no_of_ids==1?"":"s"));
57fb082000-07-09Martin Nilsson  if(!sizeof(id_pike_order)) { continue; } if(!args->nocopy)
056e052000-07-17Andreas Lange  filename += ".new"; // Create new file instead of overwriting write("-> Writing %s (%d new)", filename, sizeof(id_pike_order));
57fb082000-07-09Martin Nilsson  if(!file->open(filename, "cw")) { werror("\n* Error: Could not open %s for writing\n", filename); exit(1); } foreach(id_pike_order, array id) { // Insert ids based on tokens and the now regexp-safe string
e6a7492000-07-14Andreas Lange  object(Regexp) RE;
57fb082000-07-09Martin Nilsson  // RE = ^(.*TOKEN\( ")(", string \).*)$
7c006e2000-08-04Andreas Lange  RE = Regexp("^(.*" + id[1] + "\\([ \n\t]*)[\"0]*" + "([ ,\n\t]*"+id[2]+"[ \t\n]*\\).*)$");
e6a7492000-07-14Andreas Lange  array hits = RE->split(indata);
57fb082000-07-09Martin Nilsson  if(hits)
7c006e2000-08-04Andreas Lange  indata = hits[0] + (intp(id[0])?id[0]:"\""+id[0]+"\"") + hits[1]; else { werror("\n* Warning: Failed to set id %O for string %O in %s", id[0], ids[id[0]]->original, filename); if(sizeof(ids[id[0]]->origin)<2) id_xml_order -= ({ id[0] }); }
57fb082000-07-09Martin Nilsson  } write("\n");
056e052000-07-17Andreas Lange  file->write( indata );
57fb082000-07-09Martin Nilsson  file->truncate( file->tell() ); file->close(); } } void update_xml_sourcefiles(array filelist) { // Extracts strings from html/xml files in filelist // Updates ids, r_ids, id_xml_order with ids and strings // If new ids, updates the sourcefile or a copy foreach(filelist, string filename) {
056e052000-07-17Andreas Lange  Stdio.File file = Stdio.FILE();
57fb082000-07-09Martin Nilsson  if(!file->open(filename, "r")) { werror("* Error: Could not open sourcefile %s.\n", filename); exit(1); }
056e052000-07-17Andreas Lange  write("Reading %s", filename);
e6a7492000-07-14Andreas Lange  string line = file->gets(); string data = file->read();
57fb082000-07-09Martin Nilsson  file->close();
7c006e2000-08-04Andreas Lange  if(!data && !line)
e6a7492000-07-14Andreas Lange  continue; // Check encoding if(!line) line = data; string encoding; sscanf(line, "%*sencoding=\"%s\"", encoding); if(encoding && encoding!="") { function decode = get_decoder(encoding); if(decode && catch( data = decode(data) )) { werror("\n* Error: unable to decode from %O in %O\n", encoding, filename); exit(1); } } else if(line!=data) data = line+"\n"+data;
57fb082000-07-09Martin Nilsson  write(", parsing..."); int new = 0;
056e052000-07-17Andreas Lange  int ignoretag = 0; int no_of_ids = 0;
57fb082000-07-09Martin Nilsson  Parser.HTML xml_parser = Parser.HTML(); xml_parser->case_insensitive_tag(1);
ae23712000-08-11Andreas Lange  xml_parser->add_quote_tag("!--", lambda() {return 0;}, "--");
57fb082000-07-09Martin Nilsson  xml_parser-> add_tag("trans-reg", // Check the registertag for the right project lambda(object foo, mapping m) { if(!m->project || m->project=="") { werror("\n * Error: Missing project in %s\n", m->project, filename); exit(1); } if(args->project && m->project!=args->project)
056e052000-07-17Andreas Lange  ignoretag = 1; // Tags might be from another project
57fb082000-07-09Martin Nilsson  else
056e052000-07-17Andreas Lange  ignoretag = 0;
57fb082000-07-09Martin Nilsson  if(!args->project) args->project = m->project; return 0; }); xml_parser-> add_container("translate", // This is the string container lambda(object foo, mapping m, string c) { if(m->project && m->project!="") { if(m->project!=args->project) return 0; // Tag belongs to another project
7c006e2000-08-04Andreas Lange  // else correct project, proceed
ae23712000-08-11Andreas Lange  } else // No proj specified
7c006e2000-08-04Andreas Lange  if(ignoretag)
ae23712000-08-11Andreas Lange  return 0; // Check if last proj was another
7c006e2000-08-04Andreas Lange  string|int id = m->id; if((int)id) id = (int)id;
57fb082000-07-09Martin Nilsson  string fstr = c; int updated = 0;
056e052000-07-17Andreas Lange  if (String.trim_whites(fstr)=="")
57fb082000-07-09Martin Nilsson  return 0; // No need to store empty strings
056e052000-07-17Andreas Lange  no_of_ids++;
7c006e2000-08-04Andreas Lange  if(!id || id=="") {
57fb082000-07-09Martin Nilsson  if (r_ids[fstr]) id = r_ids[fstr]; // Re-use old id with same string else id = make_id(); // New string --> Get new id // Mark that we have a new id here updated = ++new; } else { // Verify old id
7c006e2000-08-04Andreas Lange  if(!ids[id] || (ids[id] && !ids[id]->origin)) {
57fb082000-07-09Martin Nilsson  // Remove preread string in r_ids, might be updated m_delete(r_ids, ids[id]); } else {
7c006e2000-08-04Andreas Lange  if(ids[id] && ids[id]->original!=fstr) {
57fb082000-07-09Martin Nilsson  werror("\n* Error: inconsistant use of id.\n");
7c006e2000-08-04Andreas Lange  werror(" In file:%{ %s%}\n", ids[id]->origin); werror(" id %O -> string %O\n", id, ids[id]->original);
056e052000-07-17Andreas Lange  werror(" In file: %s\n", filename); werror(" id %O -> string %O\n", id, fstr);
57fb082000-07-09Martin Nilsson  exit(1); } }
878f212000-07-10Andreas Lange  if(r_ids[fstr] && r_ids[fstr]!=id &&
7c006e2000-08-04Andreas Lange  ids[r_ids[fstr]]->origin)
878f212000-07-10Andreas Lange  werror("\n* Warning: %O has id %O in%{ %s%}, " "id %O in %s", fstr, r_ids[fstr],
7c006e2000-08-04Andreas Lange  ids[r_ids[fstr]]->origin, id, filename);
57fb082000-07-09Martin Nilsson  }
056e052000-07-17Andreas Lange  if(!has_value(id_xml_order, id))
57fb082000-07-09Martin Nilsson  // Id not in xml-structure, add to list
7c006e2000-08-04Andreas Lange  id_xml_order += ({ id }); if(!ids[id]) ids[id] = ([]); ids[id]->original = fstr; // Store id:text ids[id]->origin += ({filename}); // Add origin if(String.trim_whites(fstr)!="") r_ids[fstr] = id; // Store text:id
57fb082000-07-09Martin Nilsson  if(updated) { string ret="<translate id=\""+id+"\"";
878f212000-07-10Andreas Lange  foreach(indices(m)-({"id"}), string param) ret+=" "+param+"=\""+m[param]+"\""; return ({ ret+">"+c+"</translate>" });
57fb082000-07-09Martin Nilsson  } // Not updated, do not change return 0; });
e6a7492000-07-14Andreas Lange  xml_parser->feed(data)->finish();
57fb082000-07-09Martin Nilsson 
056e052000-07-17Andreas Lange  // Done parsing, rebuild sourcefile if needed
7c006e2000-08-04Andreas Lange  write(" (%d localization%s)\n", no_of_ids, no_of_ids==1?"":"s");
57fb082000-07-09Martin Nilsson  if(!new) { continue; }
e6a7492000-07-14Andreas Lange  data = xml_parser->read(); if(encoding && encoding!="") { function encode = get_encoder(encoding); if(encode && catch( data = encode(data) )) { werror("\n* Error: unable to encode data in %O\n", encoding); exit(1); } }
57fb082000-07-09Martin Nilsson  if(!args->nocopy)
056e052000-07-17Andreas Lange  filename += ".new"; // Create new file instead of overwriting write("-> Writing %s (%d new)", filename, new);
57fb082000-07-09Martin Nilsson  if(!file->open(filename, "cw")) { werror("\n* Error: Could not open %s for writing\n", filename); exit(1); }
e6a7492000-07-14Andreas Lange  file->write( data );
57fb082000-07-09Martin Nilsson  file->truncate( file->tell() ); file->close();
e6a7492000-07-14Andreas Lange  write("\n");
57fb082000-07-09Martin Nilsson  } } string parse_config(string filename) { // Read config in xml-format and update args([]) and files({}) // Commandline arguments have precedence // Returns name of outfile (ie project_eng.xml) if(!filename || filename=="") return "";
e6a7492000-07-14Andreas Lange  Stdio.File in=Stdio.FILE();
57fb082000-07-09Martin Nilsson  if(!in->open(filename, "r")) return "";
e6a7492000-07-14Andreas Lange  string line = in->gets();
57fb082000-07-09Martin Nilsson  string indata = in->read(); in->close();
e6a7492000-07-14Andreas Lange  if(!indata)
57fb082000-07-09Martin Nilsson  return "";
e6a7492000-07-14Andreas Lange  // Check encoding if(!line) line = indata; sscanf(line, "%*sencoding=\"%s\"", string encoding); if(encoding && encoding!="") { function decode = get_decoder(encoding); if(decode && catch( indata = decode(indata) )) { werror("\n* Error: unable to decode from %O in %O\n", encoding, filename); exit(1); } } else if(line!=indata) indata = line+"\n"+indata;
57fb082000-07-09Martin Nilsson  string xml_name=""; Parser.HTML xml_parser = Parser.HTML(); xml_parser->case_insensitive_tag(1);
ae23712000-08-11Andreas Lange  xml_parser->add_quote_tag("!--", lambda() {return 0;}, "--");
57fb082000-07-09Martin Nilsson  xml_parser-> add_container("project", // Only read config for the right project, or the // first found if unspecified lambda(object foo, mapping m, string c) { if(!m->name || m->name=="") { werror("\n* Projectname missing in %s!\n", filename); exit(1); } if(args->project && args->project!="" && args->project!=m->name) return ""; // Skip this project-tag else args->project = m->name; write("Reading config for project %O in %s\n", args->project, filename); return c; }); xml_parser-> add_container("out", // Set outname (default: project_eng.xml) lambda(object foo, mapping m, string c) { c = String.trim_whites(c); if(c && c!="") xml_name = c; return 0; }); xml_parser-> add_container("file", // Add a file to be parsed lambda(object foo, mapping m, string c) { c = String.trim_whites(c); if(c && c!="") files += ({ c }); return 0; }); xml_parser-> add_container("encoding", // Set default encoding lambda(object foo, mapping m, string c) {
e6a7492000-07-14Andreas Lange  if(args->encoding=="") args->encoding = 0;
57fb082000-07-09Martin Nilsson  c = String.trim_whites(c);
e6a7492000-07-14Andreas Lange  if(c && c!="" && !args->encoding) {
57fb082000-07-09Martin Nilsson  args->encoding = c;
e6a7492000-07-14Andreas Lange  get_encoder( c ); // Check if known }
57fb082000-07-09Martin Nilsson  return 0; }); xml_parser->
ae23712000-08-11Andreas Lange  add_container("xmlpath",
7c006e2000-08-04Andreas Lange  // Project file path
57fb082000-07-09Martin Nilsson  lambda(object foo, mapping m, string c) {
ae23712000-08-11Andreas Lange  if(!args->xmlpath) {
7c006e2000-08-04Andreas Lange  c = String.trim_whites(c);
ae23712000-08-11Andreas Lange  args->xmlpath = c;
7c006e2000-08-04Andreas Lange  } return 0; }); xml_parser-> add_container("baselang", // Project file path lambda(object foo, mapping m, string c) { if(!args->baselang) { c = String.trim_whites(c); args->baselang = c; }
57fb082000-07-09Martin Nilsson  return 0; }); xml_parser-> add_container("add", // Block to add to project-xml-files lambda(object foo, mapping m, string c) { if(!m->id || m->id=="") { werror("\n* Missing id in <add> in %s!\n", filename); exit(1); }
056e052000-07-17Andreas Lange  add[m->id] = c;
57fb082000-07-09Martin Nilsson  return 0; }); xml_parser->add_tag("nocopy", // Update the infile instead of creating infile.new lambda(object foo, mapping m) {
056e052000-07-17Andreas Lange  args->nocopy = 1;
57fb082000-07-09Martin Nilsson  return 0; }); xml_parser->add_tag("verbose", // More informative text in xml lambda(object foo, mapping m) {
056e052000-07-17Andreas Lange  args->verbose = 1;
57fb082000-07-09Martin Nilsson  return 0; }); xml_parser->add_tag("wipe", // Remove all id:strings not used in xml anymore lambda(object foo, mapping m) {
056e052000-07-17Andreas Lange  args->wipe = 1;
57fb082000-07-09Martin Nilsson  return 0; }); xml_parser->feed(indata)->finish();
7c006e2000-08-04Andreas Lange  if(xml_name=="") // Try to crate name of outfile
ae23712000-08-11Andreas Lange  if(args->xmlpath && args->baselang) xml_name = replace(args->xmlpath, "%L", args->baselang);
7c006e2000-08-04Andreas Lange  else if( args->project) xml_name = args->project+"_eng.xml";
57fb082000-07-09Martin Nilsson  return xml_name; } // ------------------------ The main program -------------------------- int main(int argc, array(string) argv) { // Parse arguments argv=argv[1..sizeof(argv)-1]; for(int i=0; i<sizeof(argv); i++) { if(argv[i][0]!='-') { files += ({argv[i]}); continue; }
056e052000-07-17Andreas Lange  string key, val = "";
57fb082000-07-09Martin Nilsson  if(sscanf(argv[i], "--%s", key)) { sscanf(key, "%s=%s", key, val);
056e052000-07-17Andreas Lange  args[key] = val;
57fb082000-07-09Martin Nilsson  continue; }
056e052000-07-17Andreas Lange  args[argv[i][1..]] = 1;
57fb082000-07-09Martin Nilsson  } // Get name of outfile (something like project_eng.xml) string xml_name=args->out; // Read configfile string configname = args->config; if(!configname && args->project) configname = args->project+".xml"; string filename = parse_config(configname);
7c006e2000-08-04Andreas Lange  if(!xml_name || xml_name=="") if(filename!="") xml_name = filename;
ae23712000-08-11Andreas Lange  else if(args->xmlpath && args->baselang) xml_name = replace(args->xmlpath, "%L", args->baselang);
7c006e2000-08-04Andreas Lange 
ae23712000-08-11Andreas Lange  if( (!(xml_name && args->sync && args->xmlpath && args->baselang)) &&
7c006e2000-08-04Andreas Lange  (!sizeof(files) || args->help) ) {
392f962002-01-16Martin Nilsson  sscanf("$Revision: 1.12 $", "$"+"Revision: %s $", string v);
57fb082000-07-09Martin Nilsson  werror("\n Locale Extractor Utility "+v+"\n\n"); werror(" Syntax: extract.pike [arguments] infile(s)\n\n"); werror(" Arguments: --project=name default: first found in infile\n"); werror(" --config=file default: [project].xml\n"); werror(" --out=file default: [project]_eng.xml\n"); werror(" --nocopy update infile instead of infile.new\n");
6dda652000-11-27Martin Nilsson  werror(" --notime don't include dump time in xml files\n");
57fb082000-07-09Martin Nilsson  werror(" --wipe remove unused ids from xml\n");
f01f742000-11-26Martin Nilsson  werror(" --sync synchronize all locale projects\n");
57fb082000-07-09Martin Nilsson  werror(" --encoding=enc default: ISO-8859-1\n"); werror(" --verbose more informative text in xml\n"); werror("\n"); return 1; } // Try to read and parse xml-file
ae23712000-08-11Andreas Lange  mapping xml_data; xml_data = parse_xml_file(xml_name, args->baselang);
7c006e2000-08-04Andreas Lange  write("\n");
57fb082000-07-09Martin Nilsson  // Read, parse and (if necessary) update the sourcefiles
056e052000-07-17Andreas Lange  object R = Regexp("(\.pike|\.pmod)$");
7c006e2000-08-04Andreas Lange  foreach(files, string filename) if(R->match(filename)) update_pike_sourcefiles( ({ filename }) ); else update_xml_sourcefiles( ({ filename }) );
f01f742000-11-26Martin Nilsson 
7c006e2000-08-04Andreas Lange  // Save all strings to outfile xml
57fb082000-07-09Martin Nilsson  if(!xml_name) if(args->project && args->project!="") xml_name = args->project+"_eng.xml"; else { xml_name = files[0]; sscanf(xml_name, "%s.pike", xml_name); xml_name += "_eng.xml"; }
7c006e2000-08-04Andreas Lange  write("\n");
ae23712000-08-11Andreas Lange  write_xml_file( xml_name, args->baselang, args->encoding || xml_data->encoding, xml_data->data);
57fb082000-07-09Martin Nilsson 
7c006e2000-08-04Andreas Lange  // Synchronize xmls in other languages if (args->sync) { write("\n"); mapping base_ids = ids; array base_order = id_xml_order;
ae23712000-08-11Andreas Lange  foreach(languagefiles(args->xmlpath, args->baselang), mapping file) {
7c006e2000-08-04Andreas Lange  ids = ([]);
ae23712000-08-11Andreas Lange  string enc = parse_xml_file(file->name, file->lang)->encoding;
7c006e2000-08-04Andreas Lange  id_xml_order = base_order; mapping old_ids = ids; ids = base_ids;
ae23712000-08-11Andreas Lange  write_xml_file(file->name, file->lang, args->encoding || enc, xml_data->data, old_ids);
7c006e2000-08-04Andreas Lange  } } write("\n");
57fb082000-07-09Martin Nilsson  return 0; }