87e9262001-06-22Martin Nilsson // This file is part of Roxen Search // Copyright © 2001 Roxen IS. All rights reserved. //
3e1a302001-07-11Johan Schön // $Id: Utils.pmod,v 1.12 2001/07/11 20:10:09 js Exp $
87e9262001-06-22Martin Nilsson 
a0fc192001-06-10Johan Schön public array(string) tokenize_and_normalize( string what )
4779302001-06-05Per Hedbor //! This can be optimized quite significantly when compared to //! tokenize( normalize( x ) ) in the future, currently it's not all //! that much faster, but still faster. { return Unicode.split_words_and_normalize( lower_case(what) ); }
90f5642001-05-17Johan Schön 
a0fc192001-06-10Johan Schön public array(string) tokenize(string in)
4779302001-06-05Per Hedbor //! Tokenize the input string (Note: You should first call normalize //! on it)
90f5642001-05-17Johan Schön {
4779302001-06-05Per Hedbor  return Unicode.split_words( in );
90f5642001-05-17Johan Schön }
a0fc192001-06-10Johan Schön public string normalize(string in)
4779302001-06-05Per Hedbor //! Normalize the input string. Performs unicode NFKD normalization //! and then lowercases the whole string
90f5642001-05-17Johan Schön {
4779302001-06-05Per Hedbor  return Unicode.normalize( lower_case(in), "KD" );
90f5642001-05-17Johan Schön }
c88b0b2001-07-04Martin Nilsson  class Logger { private string|Sql.Sql logdb; private int profile; private Sql.Sql get_db() { Sql.Sql db; #if constant(DBManager) if(stringp(logdb)) db = DBManager.get(logdb); else #endif db = logdb; // if(!logdb || !logdb->query) // throw( ({ "Couldn't find any database object.\n", backtrace() }) ); return db; } void create(string|Sql.Sql _logdb, int _profile) { logdb = _logdb; profile = _profile; // create table eventlog (event int unsigned auto_increment primary key, // at timestamp(14) not null, code int unsigned not null, extra varchar(255)) Sql.Sql db = get_db(); if(catch(db->query("SELECT code FROM eventlog WHERE event=0"))) db->query("CREATE TABLE eventlog (" "event int unsigned auto_increment primary key," "at timestamp(14) not null," "profile int unsigned not null," "code int unsigned not null," "type enum('error','warning','notice') not null," "extra varchar(255))"); }
5cc7ba2001-07-06Martin Nilsson  void log_event( int code, string type, void|string extra, void|int log_profile ) {
c88b0b2001-07-04Martin Nilsson  Sql.Sql db = get_db(); if(!db) return;
5cc7ba2001-07-06Martin Nilsson  if(zero_type(log_profile)) log_profile = profile;
c88b0b2001-07-04Martin Nilsson  if(extra) db->query("INSERT INTO eventlog (profile,code,type,extra) VALUES (%d,%d,%s,%s)",
5cc7ba2001-07-06Martin Nilsson  log_profile, code, type, extra);
c88b0b2001-07-04Martin Nilsson  else db->query("INSERT INTO eventlog (profile, code,type) VALUES (%d,%d,%s)",
5cc7ba2001-07-06Martin Nilsson  log_profile, code, type);
c88b0b2001-07-04Martin Nilsson  }
5cc7ba2001-07-06Martin Nilsson  void log_error( int code, void|string extra, void|int log_profile ) {
c88b0b2001-07-04Martin Nilsson  log_event( code, "error", extra ); }
5cc7ba2001-07-06Martin Nilsson  void log_warning( int code, void|string extra, void|int log_profile ) { log_event( code, "warning", extra, log_profile ); } void log_notice( int code, void|string extra, void|int log_profile ) { log_event( code, "notice", extra, log_profile );
c88b0b2001-07-04Martin Nilsson  }
5cc7ba2001-07-06Martin Nilsson  int add_program_name(int code, string name) { int add = search( ({ "multiprocess_crawler", "buffer_c2f", "filter", "buffer_f2i", "indexer" }), name ); if(add==-1) throw( ({ "Unknown program name \""+name+"\".\n", backtrace() }) ); return code + add;
c88b0b2001-07-04Martin Nilsson  } private mapping codes = ([
5cc7ba2001-07-06Martin Nilsson  10 : "Started crawler with %s.", 11 : "Started crawler-to-filter buffer with %s.", 12 : "Started filter with %s.", 13 : "Started filter-to-indexer buffer with %s.", 14 : "Started indexer with %s.", 20 : "Exiting crawler due to signal.", 21 : "Exiting crawler-to-filter buffer due to signal.", 22 : "Exiting filter due to signal.", 23 : "Exiting filter-to-indexer buffer due to signal.", 24 : "Exiting indexer due to signal.", 30 : "Crawler failed to set up pipe.", 31 : "Crawler-to-filter buffer failed to set up pipe.", 32 : "Filter failed to set up pipe.", 33 : "Filter-to-indexer buffer failed to set up pipe.", 34 : "Indexer failed to set up pipe.", 40 : "Fetched %s.", 41 : "Unknown language code \"%s\".", 42 : "Crawler exited normally.", 43 : "Cleared search database.",
3e1a302001-07-11Johan Schön  44 : "Sitebuilder commit triggered indexing of %s.",
5cc7ba2001-07-06Martin Nilsson  50 : "Crawler did not get any connection from the process.", 51 : "Crawler-to-filter bufferdid not get any connection from the process.", 52 : "Filter did not get any connection from the process.", 53 : "Filter-to-indexer buffer did not get any connection from the process.", 54 : "Indexer did not get any connection from the process.",
c88b0b2001-07-04Martin Nilsson  ]);
7d85aa2001-07-04Martin Nilsson  array(array(string|int)) get_log( int profile, array(string) types, int from, int to ) {
c88b0b2001-07-04Martin Nilsson  string sql = "";
5990902001-07-05Martin Nilsson #define SQLADD(X) do{sizeof(sql)?(sql+=" AND "+(X)):(sql=" WHERE "+(X));}while(0)
c88b0b2001-07-04Martin Nilsson  if(profile) SQLADD("profile=" + profile); if(!sizeof(types)) return ({}); if(sizeof(types)!=3) SQLADD("(type='" + (types*"' OR type='") + "')"); if(from) SQLADD("at>" + from); if(to) SQLADD("to<" + to); #undef SQLADD Sql.Sql db = get_db();
7d85aa2001-07-04Martin Nilsson  if(!db) return ({});
c88b0b2001-07-04Martin Nilsson 
2cd10c2001-07-05Martin Nilsson  return map(db->query("SELECT unix_timestamp(at) as at,profile,code,type,extra FROM eventlog" + sql + " ORDER BY event DESC"),
c88b0b2001-07-04Martin Nilsson  lambda(mapping in) {
7d85aa2001-07-04Martin Nilsson  return ({ (int)in->at, (int)in->profile, in->type,
2cd10c2001-07-05Martin Nilsson  in->extra?sprintf(codes[(int)in->code], @(in->extra/"\n")): codes[(int)in->code] });
c88b0b2001-07-04Martin Nilsson  } ); } }