87e9262001-06-22Martin Nilsson // This file is part of Roxen Search
6351652009-05-25Martin Stjernholm // Copyright © 2001 - 2009, Roxen IS. All rights reserved.
87e9262001-06-22Martin Nilsson //
de9a7c2010-10-06Henrik Grubbström (Grubba) // $Id$
f672612001-07-26Martin Nilsson  #if !constant(report_error) #define report_error werror
2db1ff2001-09-13Martin Nilsson #define report_debug werror #define report_warning werror #endif
de9a7c2010-10-06Henrik Grubbström (Grubba) final constant dont_dump_module=1;
2db1ff2001-09-13Martin Nilsson #ifdef SEARCH_DEBUG # define WERR(X) report_debug("search: "+(X)+"\n"); #else # define WERR(X)
f672612001-07-26Martin Nilsson #endif
87e9262001-06-22Martin Nilsson 
a0fc192001-06-10Johan Schön public array(string) tokenize_and_normalize( string what )
4779302001-06-05Per Hedbor //! This can be optimized quite significantly when compared to //! tokenize( normalize( x ) ) in the future, currently it's not all //! that much faster, but still faster. { return Unicode.split_words_and_normalize( lower_case(what) ); }
90f5642001-05-17Johan Schön 
a0fc192001-06-10Johan Schön public array(string) tokenize(string in)
4779302001-06-05Per Hedbor //! Tokenize the input string (Note: You should first call normalize //! on it)
90f5642001-05-17Johan Schön {
4779302001-06-05Per Hedbor  return Unicode.split_words( in );
90f5642001-05-17Johan Schön }
a0fc192001-06-10Johan Schön public string normalize(string in)
4779302001-06-05Per Hedbor //! Normalize the input string. Performs unicode NFKD normalization //! and then lowercases the whole string
90f5642001-05-17Johan Schön {
4779302001-06-05Per Hedbor  return Unicode.normalize( lower_case(in), "KD" );
90f5642001-05-17Johan Schön }
c88b0b2001-07-04Martin Nilsson 
74de682001-07-13Martin Nilsson 
46a6522001-08-06Martin Nilsson //! A result entry from the @[ProfileCache].
8898352001-07-21Martin Nilsson class ProfileEntry {
74de682001-07-13Martin Nilsson 
8898352001-07-21Martin Nilsson  private int last_stat; private int database_profile_id;
47d8e92001-08-19Martin Nilsson  private int query_profile_id;
8898352001-07-21Martin Nilsson  private ProfileCache my_cache; private mapping(string:mixed) database_values;
47d8e92001-08-19Martin Nilsson  private mapping(string:mixed) query_values;
8898352001-07-21Martin Nilsson  private Search.Database.MySQL db; private Search.RankingProfile ranking; private array(string) stop_words;
47d8e92001-08-19Martin Nilsson  //! @decl void create(int database_profile_id, int query_profile_id,@
46a6522001-08-06Martin Nilsson  //! ProfileCache cache) //! @param cache //! The parent cache object.
8898352001-07-21Martin Nilsson  void create(int _database_profile_id,
47d8e92001-08-19Martin Nilsson  int _query_profile_id,
8898352001-07-21Martin Nilsson  ProfileCache _my_cache) { database_profile_id = _database_profile_id;
47d8e92001-08-19Martin Nilsson  query_profile_id = _query_profile_id;
8898352001-07-21Martin Nilsson  my_cache = _my_cache;
db9a632002-02-20Johan Schön  int last_stat = time();
f501ee2001-08-14Martin Nilsson  // Prefetch.. get_ranking();
8898352001-07-21Martin Nilsson  }
f501ee2001-08-14Martin Nilsson  //! Checks if it is time to check if the profile values are //! to old. int(0..1) check_timeout() {
db9a632002-02-20Johan Schön  if(time()-last_stat < 5*60) return 0; last_stat = time();
f501ee2001-08-14Martin Nilsson  return 1;
8898352001-07-21Martin Nilsson  }
46a6522001-08-06Martin Nilsson  //! Returns the database profile value @[index].
8898352001-07-21Martin Nilsson  mixed get_database_value(string index) { if(!database_values) database_values = my_cache->get_value_mapping(database_profile_id); return database_values[index]; }
47d8e92001-08-19Martin Nilsson  //! Returns the query profile value @[index]. mixed get_query_value(string index) { if(!query_values) query_values = my_cache->get_value_mapping(query_profile_id); return query_values[index];
8898352001-07-21Martin Nilsson  }
46a6522001-08-06Martin Nilsson  //! Returns a cached search database for the current database profile.
8898352001-07-21Martin Nilsson  Search.Database.MySQL get_database() { if(!db) {
f672612001-07-26Martin Nilsson #if constant(DBManager)
8898352001-07-21Martin Nilsson  db = Search.Database.MySQL( DBManager.db_url( get_database_value("db_name"), 1) );
f672612001-07-26Martin Nilsson #endif
8898352001-07-21Martin Nilsson  if(!db)
a65d562004-09-20Martin Stjernholm  error("Could not aquire the database URL to database " +
8898352001-07-21Martin Nilsson  get_database_value("db_name") + ".\n"); } return db; }
46a6522001-08-06Martin Nilsson  //! Returns a cached ranking profile for the current database and
47d8e92001-08-19Martin Nilsson  //! query profile.
8898352001-07-21Martin Nilsson  Search.RankingProfile get_ranking() { if(!ranking)
47d8e92001-08-19Martin Nilsson  ranking = Search.RankingProfile(get_query_value("fi_cut"), get_query_value("px_rank"),
8898352001-07-21Martin Nilsson  get_database(),
47d8e92001-08-19Martin Nilsson  get_query_value("fi_rank"));
8898352001-07-21Martin Nilsson  return ranking; } class ADTSet { private mapping vals = ([]); ADTSet add (string|int|float in) { vals[in] = 1; return this_object(); } ADTSet sub (string|int|float out) { m_delete(vals, out); return this_object(); } ADTSet `+(mixed in) { if(stringp(in)||intp(in)||floatp(in)) add(in); else map((array)in, add); return this_object(); } ADTSet `-(mixed out) { if(stringp(out)||intp(out)||floatp(out)) sub(out); else map((array)out, sub); return this_object(); } mixed cast(string to) { switch(to) { case "object": return this_object(); case "array": return indices(vals); case "multiset": return (multiset)indices(vals); default:
a65d562004-09-20Martin Stjernholm  error("Can not cast ADTSet to "+to+".\n");
8898352001-07-21Martin Nilsson  } } }
47d8e92001-08-19Martin Nilsson  //! Returns a cached array of stop words for the current query profile.
8898352001-07-21Martin Nilsson  array(string) get_stop_words() { if(!stop_words) { ADTSet words = ADTSet();
47d8e92001-08-19Martin Nilsson  foreach(get_query_value("sw_lists"), string fn) {
8898352001-07-21Martin Nilsson  string file = Stdio.read_file(fn); if(!fn) report_error("Could not load %O.\n", fn); else words + (Array.flatten(map(file/"\n", lambda(string in) { return in/" "; }))-({""})); }
47d8e92001-08-19Martin Nilsson  words + (Array.flatten(map(get_query_value("sw_words")/"\n",
8898352001-07-21Martin Nilsson  lambda(string in) { return in/" "; }))-({""})); stop_words = (array)words; } return stop_words; } }
46a6522001-08-06Martin Nilsson //!
8898352001-07-21Martin Nilsson class ProfileCache (string db_name) {
f501ee2001-08-14Martin Nilsson  private mapping(string:ProfileEntry) entry_cache = ([]);
74de682001-07-13Martin Nilsson  private mapping(int:mapping(string:mixed)) value_cache = ([]); private mapping(string:int) db_profile_names = ([]);
47d8e92001-08-19Martin Nilsson  private mapping(string:int) query_profile_names = ([]);
8898352001-07-21Martin Nilsson  private mapping(int:int) profile_stat = ([]);
74de682001-07-13Martin Nilsson  private Sql.Sql get_db() {
f672612001-07-26Martin Nilsson  Sql.Sql db; #if constant(DBManager) db = DBManager.cached_get(db_name); #endif
a65d562004-09-20Martin Stjernholm  if(!db) error("Could not connect to database " + db_name + ".\n");
74de682001-07-13Martin Nilsson  return db; }
46a6522001-08-06Martin Nilsson  //! Checks if the profile @[profile_id] has been changed, and clears //! related caches if so. //! @returns //! @int //! @value -1 //! The profile is deleted. //! @value 0
f501ee2001-08-14Martin Nilsson  //! The profile is not up to date.
46a6522001-08-06Martin Nilsson  //! @value 1
f501ee2001-08-14Martin Nilsson  //! The profile is up to date.
46a6522001-08-06Martin Nilsson  //! @endint
8898352001-07-21Martin Nilsson  int(-1..1) up_to_datep(int profile_id) { array(mapping(string:string)) res;
edd6132001-08-22Martin Nilsson  res = get_db()->query("SELECT altered,type FROM profile WHERE id=%d", profile_id);
8898352001-07-21Martin Nilsson  // The profile is deleted. In such a rare event we take the // trouble to review all our cached values. if(!sizeof(res)) {
edd6132001-08-22Martin Nilsson  array(int) existing = (array(int))get_db()->query("SELECT id FROM profile")->id;
8898352001-07-21Martin Nilsson  foreach(indices(value_cache), int id) if(!has_value(existing, id)) m_delete(value_cache, id);
f501ee2001-08-14Martin Nilsson  foreach(indices(entry_cache), string id) {
47d8e92001-08-19Martin Nilsson  int dbp, qp; sscanf(id, "%d:%d", dbp, qp);
f501ee2001-08-14Martin Nilsson  if(!has_value(existing, dbp)) m_delete(entry_cache, id);
47d8e92001-08-19Martin Nilsson  if(!has_value(existing, qp))
8898352001-07-21Martin Nilsson  m_delete(entry_cache, id);
f501ee2001-08-14Martin Nilsson  }
8898352001-07-21Martin Nilsson  foreach(indices(db_profile_names), string name) if(!has_value(existing, db_profile_names[name])) m_delete(db_profile_names, name);
47d8e92001-08-19Martin Nilsson  foreach(indices(query_profile_names), string name) if(!has_value(existing, query_profile_names[name])) m_delete(query_profile_names, name);
8898352001-07-21Martin Nilsson  return -1; } // Not altered if((int)res[0]->altered == profile_stat[profile_id]) return 1;
3208782001-07-26Martin Nilsson  profile_stat[profile_id] = (int)res[0]->altered;
8898352001-07-21Martin Nilsson 
47d8e92001-08-19Martin Nilsson  // Query profile
2208892001-08-09Per Hedbor  if((int)res[0]->type == 2) {
8898352001-07-21Martin Nilsson  m_delete(value_cache, profile_id);
f501ee2001-08-14Martin Nilsson  foreach(indices(entry_cache), string id) if(array_sscanf(id, "%d:%d")[1]==profile_id) m_delete(entry_cache, id);
8898352001-07-21Martin Nilsson  return 0; } m_delete(value_cache, profile_id);
f501ee2001-08-14Martin Nilsson  foreach(indices(entry_cache), string id) if(array_sscanf(id, "%d:%d")[0]==profile_id) m_delete(entry_cache, id);
8898352001-07-21Martin Nilsson  return 0; }
46a6522001-08-06Martin Nilsson  //! Returns the profile number for the given database profile.
74de682001-07-13Martin Nilsson  int get_db_profile_number(string name) { int db_profile; if(db_profile=db_profile_names[name]) return db_profile; array res = get_db()->
edd6132001-08-22Martin Nilsson  query("SELECT id FROM profile WHERE name=%s AND type=2", name);
74de682001-07-13Martin Nilsson  if(!sizeof(res))
a65d562004-09-20Martin Stjernholm  error("No database profile " + name + " found.\n");
74de682001-07-13Martin Nilsson  return db_profile_names[name] = (int)res[0]->id; }
47d8e92001-08-19Martin Nilsson  //! Returns the profile number for the given query profile. int get_query_profile_number(string name)
2208892001-08-09Per Hedbor  {
47d8e92001-08-19Martin Nilsson  int query_profile; if( query_profile=query_profile_names[name] ) return query_profile;
74de682001-07-13Martin Nilsson  array res = get_db()->
edd6132001-08-22Martin Nilsson  query("SELECT id FROM profile WHERE name=%s AND type=1", name);
74de682001-07-13Martin Nilsson  if(!sizeof(res))
a65d562004-09-20Martin Stjernholm  error("No query profile " + name + " found.\n");
74de682001-07-13Martin Nilsson 
47d8e92001-08-19Martin Nilsson  return query_profile_names[name] = (int)res[0]->id;
74de682001-07-13Martin Nilsson  }
23a6582001-07-31David Norlin  private int last_db_prof_stat = 0; // 1970
46a6522001-08-06Martin Nilsson  //! Returns a list of available database profiles.
23a6582001-07-31David Norlin  array(string) list_db_profiles() {
dddfff2001-11-22Johan Schön  /*
db9a632002-02-20Johan Schön  if (time() - last_db_prof_stat < 5*60)
dddfff2001-11-22Johan Schön  return indices(db_profile_names);*/
edd6132001-08-22Martin Nilsson  array res = get_db()->query("SELECT name, id FROM profile WHERE type=2");
23a6582001-07-31David Norlin  db_profile_names = mkmapping( res->name, map(res->id, lambda(string s) { return (int) s; } ));
fbf68c2001-08-16Martin Nilsson  if(sizeof(res))
db9a632002-02-20Johan Schön  last_db_prof_stat = time();
23a6582001-07-31David Norlin  return res->name; }
47d8e92001-08-19Martin Nilsson  private int last_query_prof_stat = 0; // 1970
46a6522001-08-06Martin Nilsson 
47d8e92001-08-19Martin Nilsson  //! Returns a list of available query profiles. array(string) list_query_profiles()
2208892001-08-09Per Hedbor  {
dddfff2001-11-22Johan Schön  /*
db9a632002-02-20Johan Schön  if (time() - last_query_prof_stat < 5*60)
dddfff2001-11-22Johan Schön  return indices(query_profile_names);*/ array res = get_db()->query("SELECT name, id FROM profile WHERE type=1"); query_profile_names = mkmapping( res->name, (array(int)) res->id ); if(sizeof(query_profile_names))
db9a632002-02-20Johan Schön  last_query_prof_stat = time();
23a6582001-07-31David Norlin  }
74de682001-07-13Martin Nilsson  // Used when decoding text encoded pike data types. private object compile_handler = class { mapping(string:mixed) get_default_module() { return ([ "aggregate_mapping":aggregate_mapping, "aggregate_multiset":aggregate_multiset, "aggregate":aggregate, "allocate":allocate,
c228292010-05-18Martin Stjernholm  "`+": `+,
74de682001-07-13Martin Nilsson  "this_program":0 ]); } mixed resolv(string id, void|string fn, void|string ch) { throw( ({ sprintf("Found symbol %O while trying to decode Roxen Search " "settings. The database is corrupt or has been " "tampered with.\n", id), backtrace() }) ); } }(); // Decodes a "readable" pike data type to its actual pike value. private mixed reacodec_decode(string str) { return compile_string("mixed foo=" + str + ";", 0, compile_handler)()->foo; }
46a6522001-08-06Martin Nilsson  //! Returns the value mapping for the given profile.
74de682001-07-13Martin Nilsson  mapping get_value_mapping(int profile) { mapping val; if(val=copy_value(value_cache[profile])) return val; array res = get_db()->
edd6132001-08-22Martin Nilsson  query("SELECT name,value FROM value WHERE pid=%d", profile);
74de682001-07-13Martin Nilsson  val = mkmapping( res->name, map(res->value, reacodec_decode) ); value_cache[profile] = copy_value(val); return val; }
46a6522001-08-06Martin Nilsson  //! Returns a @[ProfileEntry] object with the states needed for //! commiting searches in the database profile @[db_name] with
47d8e92001-08-19Martin Nilsson  //! the rules from query profile @[query_name]. ProfileEntry get_profile_entry(string db_name, void|string query_name) {
3208782001-07-26Martin Nilsson  // werror("Entry: %O\n", indices(entry_cache)); // werror("Value: %O\n", indices(value_cache)); // werror("Stat : %O\n", profile_stat);
74de682001-07-13Martin Nilsson  int db = get_db_profile_number(db_name);
47d8e92001-08-19Martin Nilsson  int query = get_query_profile_number(query_name);
8898352001-07-21Martin Nilsson  ProfileEntry entry;
47d8e92001-08-19Martin Nilsson  if(entry=entry_cache[query +":"+ db]) {
f501ee2001-08-14Martin Nilsson  if(!entry->check_timeout()) return entry; if(up_to_datep(db) &&
47d8e92001-08-19Martin Nilsson  up_to_datep(query)) return entry;
f501ee2001-08-14Martin Nilsson  }
8898352001-07-21Martin Nilsson 
47d8e92001-08-19Martin Nilsson  entry = ProfileEntry( db, query, this_object() ); return entry_cache[query +":"+ db] = entry;
74de682001-07-13Martin Nilsson  }
46a6522001-08-06Martin Nilsson  //! Flushes profile entry @[p] from the profile cache.
74de682001-07-13Martin Nilsson  void flush_profile(int p) { m_delete(value_cache, p);
4abea02008-06-24Jonas Wallden  foreach(db_profile_names; string name; int dbp) if (dbp == p)
74de682001-07-13Martin Nilsson  m_delete(db_profile_names, name);
4abea02008-06-24Jonas Wallden  foreach(query_profile_names; string name; int qp) if (qp == p) m_delete(query_profile_names, name);
f501ee2001-08-14Martin Nilsson  foreach(indices(entry_cache), string id) { array ids = array_sscanf(id, "%d:%d"); if(ids[0]==p || ids[1]==p) m_delete(entry_cache, id); }
74de682001-07-13Martin Nilsson  } //! Empty the whole cache. void flush_cache() { value_cache = ([]); db_profile_names = ([]);
47d8e92001-08-19Martin Nilsson  query_profile_names = ([]);
f82b842001-08-31Johan Schön  last_db_prof_stat = 0; last_query_prof_stat = 0;
74de682001-07-13Martin Nilsson  } }
f501ee2001-08-14Martin Nilsson private mapping(string:ProfileCache) profile_cache_cache = ([]);
2db1ff2001-09-13Martin Nilsson //! Returns a @[ProfileCache] for the profiles stored //! in the database @[db_name].
f501ee2001-08-14Martin Nilsson ProfileCache get_profile_cache(string db_name) { if(profile_cache_cache[db_name]) return profile_cache_cache[db_name]; return profile_cache_cache[db_name] = ProfileCache(db_name); }
2db1ff2001-09-13Martin Nilsson //! Flushes the profile @[p] from all @[ProfileCache] objects //! obtained with @[get_profile_cache].
f501ee2001-08-14Martin Nilsson void flush_profile(int p) { values(profile_cache_cache)->flush_profile(p); }
2db1ff2001-09-13Martin Nilsson private mapping(string:mapping) profile_storages = ([]); //! Returns a profile storage mapping, which will be shared //! between all callers with the same @[db_name] given. mapping get_profile_storage(string db_name) { if(profile_storages[db_name]) return profile_storages[db_name]; return profile_storages[db_name] = ([]); } private mapping(string:Scheduler) scheduler_storage = ([]); //! Returns a scheduler for the given profile database. Scheduler get_scheduler(string db_name) { mapping dbp = get_profile_storage(db_name); if(scheduler_storage[db_name]) return scheduler_storage[db_name]; scheduler_storage[db_name] = Scheduler(dbp); return scheduler_storage[db_name] = Scheduler(dbp); } class Scheduler { private int next_run;
5808772004-03-08Anders Johansson  private mapping(int:int) entry_queue = ([]); private mapping(int:int) crawl_queue = ([]); private mapping(int:int) compact_queue = ([]); private array(int) priority_queue = ({});
2db1ff2001-09-13Martin Nilsson  private mapping db_profiles;
916f662008-08-20Jonas Wallden #if constant(roxen)
27d1622010-06-03Fredrik Noring  private mixed schedule_run;
916f662008-08-20Jonas Wallden #endif
2db1ff2001-09-13Martin Nilsson  void create(mapping _db_profiles) { db_profiles = _db_profiles; schedule(); }
5808772004-03-08Anders Johansson  void check_priority_queue(int profile) { if (!has_value(priority_queue, profile)) priority_queue += ({ profile }); }
2db1ff2001-09-13Martin Nilsson  //! Call this method to indicate that a new entry has been added //! to the queue. The scheduler will delay indexing with at most //! @[latency] minutes. void new_entry(int latency, array(int) profiles) { int would_be_indexed = time() + latency*60; foreach(profiles, int profile)
5808772004-03-08Anders Johansson  {
1c17cd2003-08-14Mattias Andersson  entry_queue[profile] = 0;
5808772004-03-08Anders Johansson  check_priority_queue(profile); } WERR("New entry. time: "+(would_be_indexed-time())+" profiles: "+ (array(string))profiles*",");
db9a632002-02-20Johan Schön  if(next_run && next_run<would_be_indexed && next_run>=time())
2db1ff2001-09-13Martin Nilsson  return; next_run = would_be_indexed; reschedule(); }
5808772004-03-08Anders Johansson  void schedule(void|int quiet) {
2db1ff2001-09-13Martin Nilsson  foreach(indices(db_profiles), int id) { object dbp = db_profiles[id]; if(!dbp) { report_warning("Search database profile %d destructed.\n", id); m_delete(db_profiles, id); continue; }
5808772004-03-08Anders Johansson  if(!quiet) WERR("Scheduling for database profile "+dbp->name); int next = dbp->next_recrawl();
2db1ff2001-09-13Martin Nilsson  if(next != -1) { crawl_queue[dbp->id] = next;
5808772004-03-08Anders Johansson  check_priority_queue(id); if(!quiet) WERR(" Crawl: "+(next-time()));
2db1ff2001-09-13Martin Nilsson  } next = dbp->next_compact(); if(next != -1) { compact_queue[dbp->id] = next;
5808772004-03-08Anders Johansson  if(!quiet) WERR(" Compact: "+(next-time()));
2db1ff2001-09-13Martin Nilsson  }
5808772004-03-08Anders Johansson  if(!quiet) WERR("");
2db1ff2001-09-13Martin Nilsson  }
5808772004-03-08Anders Johansson  if(!sizeof(crawl_queue) && !sizeof(compact_queue) && !sizeof(entry_queue)) return; next_run = max( min( @values(crawl_queue) + values(compact_queue) + values(entry_queue) ), time() + 10 );
2db1ff2001-09-13Martin Nilsson  reschedule(); }
8a06e92003-01-27Mattias Andersson #if constant (roxen) private void reschedule() { WERR("Scheduler runs next event in "+(next_run-time())+" seconds.");
27d1622010-06-03Fredrik Noring  remove_call_out(schedule_run); schedule_run = roxen.background_run(next_run-time(), do_scheduled_stuff);
8a06e92003-01-27Mattias Andersson  } void unschedule() {
27d1622010-06-03Fredrik Noring  remove_call_out(schedule_run); schedule_run = 0;
8a06e92003-01-27Mattias Andersson  } private void do_scheduled_stuff() {
27d1622010-06-03Fredrik Noring  remove_call_out(schedule_run); schedule_run = 0;
8a06e92003-01-27Mattias Andersson  WERR("Running scheduler event.");
5808772004-03-08Anders Johansson  foreach(indices(db_profiles), int id) { if (db_profiles[id]->is_running()) { WERR("Postponing crawl start, profile "+id+" still running."); schedule(1); return; } }
8a06e92003-01-27Mattias Andersson  int t = time(); WERR(sizeof(crawl_queue)+" profiles in crawl queue.");
5808772004-03-08Anders Johansson  foreach(priority_queue & indices(crawl_queue), int id) {
8a06e92003-01-27Mattias Andersson  if(crawl_queue[id]>t || !db_profiles[id]) continue; object dbp = db_profiles[id];
1c17cd2003-08-14Mattias Andersson  if(dbp && dbp->ready_to_crawl()) { WERR("Scheduler starts crawling "+id);
593c0e2003-08-15Mattias Andersson  dbp->recrawl();
5808772004-03-08Anders Johansson  m_delete(crawl_queue, id); m_delete(entry_queue, id); priority_queue -= ({ id });
1c17cd2003-08-14Mattias Andersson  } } WERR(sizeof(entry_queue)+" profiles in entry queue.");
5808772004-03-08Anders Johansson  foreach(priority_queue & indices(entry_queue), int id) {
1c17cd2003-08-14Mattias Andersson  if(entry_queue[id]>t || !db_profiles[id]) continue; object dbp = db_profiles[id];
8a06e92003-01-27Mattias Andersson  if(dbp && dbp->ready_to_crawl()) { WERR("Scheduler starts crawling "+id); dbp->start_indexer();
5808772004-03-08Anders Johansson  m_delete(entry_queue, id); priority_queue -= ({ id }); break;
8a06e92003-01-27Mattias Andersson  } } WERR(sizeof(compact_queue)+" profiles in compact queue."); foreach(indices(compact_queue), int id) { if(compact_queue[id]>t || !db_profiles[id]) continue; db_profiles[id]->start_compact();
5808772004-03-08Anders Johansson  m_delete(compact_queue, id);
8a06e92003-01-27Mattias Andersson  } schedule(); } #else private void reschedule() { remove_call_out(do_scheduled_stuff); WERR("Scheduler runs next event in "+(next_run-time())+" seconds."); call_out(do_scheduled_stuff, next_run-time()); } void unschedule() { remove_call_out(do_scheduled_stuff); }
2db1ff2001-09-13Martin Nilsson  private void do_scheduled_stuff() { remove_call_out(do_scheduled_stuff); WERR("Running scheduler event."); int t = time(); WERR(sizeof(crawl_queue)+" profiles in crawl queue."); foreach(indices(crawl_queue), int id) {
593c0e2003-08-15Mattias Andersson  if(crawl_queue[id]>t || !db_profiles[id]) continue; object dbp = db_profiles[id]; if(dbp && dbp->ready_to_crawl()) { WERR("Scheduler starts crawling "+id); dbp->recrawl(); entry_queue = ([]); } } WERR(sizeof(crawl_queue)+" profiles in crawl queue."); foreach(indices(crawl_queue), int id) {
2db1ff2001-09-13Martin Nilsson  if(crawl_queue[id]>t || !db_profiles[id]) continue; object dbp = db_profiles[id]; if(dbp && dbp->ready_to_crawl()) { WERR("Scheduler starts crawling "+id); dbp->start_indexer(); } } WERR(sizeof(compact_queue)+" profiles in compact queue."); foreach(indices(compact_queue), int id) { if(compact_queue[id]>t || !db_profiles[id]) continue; db_profiles[id]->start_compact(); } schedule(); }
8a06e92003-01-27Mattias Andersson #endif
2db1ff2001-09-13Martin Nilsson  string info() { string res = "<table border='1' cellspacing='0' cellpadding='2'>" "<tr><th>Profile</th><th>Crawl</th>" "<th>Compact</th><th>Next</th></tr>"; foreach(values(db_profiles), object dbp) { if(!dbp) continue; res += "</tr><td>" + dbp->name + "</td>"; int next = dbp->next_crawl(); if(next == -1) res += "<td>Never</td>"; else res +="<td>"+ (next-time()) + "</td>"; next = dbp->next_compact(); if(next == -1) res += "<td>Never</td>"; else res +="<td>"+ (next-time()) + "</td>"; res += "</tr>"; } res += "</table>"; res += "<br />Next run: " + (next_run-time()) + "<br />"; return res; } }
f501ee2001-08-14Martin Nilsson 
46a6522001-08-06Martin Nilsson //!
c88b0b2001-07-04Martin Nilsson class Logger { private string|Sql.Sql logdb; private int profile;
dbde022002-03-07Johan Schön  private int stderr_logging;
c88b0b2001-07-04Martin Nilsson 
f2b5962010-04-28Fredrik Noring  private int last_log_purge_time; private constant log_purge_freq = 8*60*60; // Purge log every 8 h or so.
c88b0b2001-07-04Martin Nilsson  private Sql.Sql get_db() { Sql.Sql db; #if constant(DBManager) if(stringp(logdb)) db = DBManager.get(logdb); else #endif db = logdb; // if(!logdb || !logdb->query) // throw( ({ "Couldn't find any database object.\n", backtrace() }) ); return db; }
dbde022002-03-07Johan Schön  //! @decl void create(Sql.Sql db_object, int profile, int stderr_logging) //! @decl void create(string db_url, int profile, int stderr_logging) void create(string|Sql.Sql _logdb, int _profile, int _stderr_logging) {
c88b0b2001-07-04Martin Nilsson  logdb = _logdb; profile = _profile;
dbde022002-03-07Johan Schön  stderr_logging = _stderr_logging;
c88b0b2001-07-04Martin Nilsson  // create table eventlog (event int unsigned auto_increment primary key, // at timestamp(14) not null, code int unsigned not null, extra varchar(255)) Sql.Sql db = get_db(); if(catch(db->query("SELECT code FROM eventlog WHERE event=0"))) db->query("CREATE TABLE eventlog (" "event int unsigned auto_increment primary key," "at timestamp(14) not null," "profile int unsigned not null," "code int unsigned not null," "type enum('error','warning','notice') not null," "extra varchar(255))"); }
dbde022002-03-07Johan Schön  void werror_event( int code, string type, void|string extra, void|int log_profile ) { mapping types = ([ "error" : "Error", "warning" : "Warning", "notice" : "Notice", ]); werror(sprintf("%sSearch: %s: %s\n", " : ", types[type], extra?sprintf(codes[(int)code], @(extra/"\n")):codes[(int)code])); }
6e1a6b2007-05-16Fredrik Noring  void log_purge(int days) { Sql.Sql db = get_db(); if(!db) return;
f2b5962010-04-28Fredrik Noring  int t1 = time(); if (t1 < last_log_purge_time + log_purge_freq) return; last_log_purge_time = t1;
6e1a6b2007-05-16Fredrik Noring  if(days) db->query("DELETE FROM eventlog " " WHERE at <= NOW() - INTERVAL "+days+" DAY"); else
f2b5962010-04-28Fredrik Noring  db->query("DELETE FROM eventlog"); // FIXME: TRUNCATE TABLE? db->query("OPTIMIZE TABLE eventlog"); int t2 = time(); if (t2 - t1 > 10) // More than 10 s is somewhat slow, report warning. report_warning("Search log purge took %d s.\n", t2-t1);
6e1a6b2007-05-16Fredrik Noring  }
dbde022002-03-07Johan Schön  //!
5cc7ba2001-07-06Martin Nilsson  void log_event( int code, string type, void|string extra, void|int log_profile ) {
c88b0b2001-07-04Martin Nilsson  Sql.Sql db = get_db(); if(!db) return;
5cc7ba2001-07-06Martin Nilsson  if(zero_type(log_profile)) log_profile = profile;
dbde022002-03-07Johan Schön  if(stderr_logging) werror_event(code, type, extra, log_profile);
c88b0b2001-07-04Martin Nilsson  if(extra) db->query("INSERT INTO eventlog (profile,code,type,extra) VALUES (%d,%d,%s,%s)",
5cc7ba2001-07-06Martin Nilsson  log_profile, code, type, extra);
c88b0b2001-07-04Martin Nilsson  else db->query("INSERT INTO eventlog (profile, code,type) VALUES (%d,%d,%s)",
5cc7ba2001-07-06Martin Nilsson  log_profile, code, type);
c88b0b2001-07-04Martin Nilsson  }
dbde022002-03-07Johan Schön 
46a6522001-08-06Martin Nilsson  //!
5cc7ba2001-07-06Martin Nilsson  void log_error( int code, void|string extra, void|int log_profile ) {
46a6522001-08-06Martin Nilsson  log_event( code, "error", extra, log_profile );
c88b0b2001-07-04Martin Nilsson  }
46a6522001-08-06Martin Nilsson  //!
5cc7ba2001-07-06Martin Nilsson  void log_warning( int code, void|string extra, void|int log_profile ) { log_event( code, "warning", extra, log_profile ); }
46a6522001-08-06Martin Nilsson  //!
5cc7ba2001-07-06Martin Nilsson  void log_notice( int code, void|string extra, void|int log_profile ) { log_event( code, "notice", extra, log_profile );
c88b0b2001-07-04Martin Nilsson  }
46a6522001-08-06Martin Nilsson  //!
5cc7ba2001-07-06Martin Nilsson  int add_program_name(int code, string name) { int add = search( ({ "multiprocess_crawler", "buffer_c2f", "filter", "buffer_f2i", "indexer" }), name ); if(add==-1) throw( ({ "Unknown program name \""+name+"\".\n", backtrace() }) ); return code + add;
c88b0b2001-07-04Martin Nilsson  } private mapping codes = ([
5cc7ba2001-07-06Martin Nilsson  10 : "Started crawler with %s.", 11 : "Started crawler-to-filter buffer with %s.", 12 : "Started filter with %s.", 13 : "Started filter-to-indexer buffer with %s.", 14 : "Started indexer with %s.", 20 : "Exiting crawler due to signal.", 21 : "Exiting crawler-to-filter buffer due to signal.", 22 : "Exiting filter due to signal.", 23 : "Exiting filter-to-indexer buffer due to signal.", 24 : "Exiting indexer due to signal.", 30 : "Crawler failed to set up pipe.", 31 : "Crawler-to-filter buffer failed to set up pipe.", 32 : "Filter failed to set up pipe.", 33 : "Filter-to-indexer buffer failed to set up pipe.", 34 : "Indexer failed to set up pipe.", 40 : "Fetched %s.", 41 : "Unknown language code \"%s\".", 42 : "Crawler exited normally.", 43 : "Cleared search database.",
3e1a302001-07-11Johan Schön  44 : "Sitebuilder commit triggered indexing of %s.",
5cc7ba2001-07-06Martin Nilsson  50 : "Crawler did not get any connection from the process.",
11b2ed2007-03-15Jonas Wallden  51 : "Crawler-to-filter buffer did not get any connection from the process.",
5cc7ba2001-07-06Martin Nilsson  52 : "Filter did not get any connection from the process.", 53 : "Filter-to-indexer buffer did not get any connection from the process.", 54 : "Indexer did not get any connection from the process.",
c253d42001-07-12Johan Schön 
26d75c2001-08-08Per Hedbor  60 : "Starting database compactor with %s", 61 : "Failed to find any data in the database.", 62 : "Exiting compacter due to signal.", 63 : "Done with database compacting and maintenance.",
c253d42001-07-12Johan Schön  300: "300 Redirection: Multiple Choices (%s)", 301: "301 Redirection: Moved Permanently (%s)", 302: "302 Redirection: Found (%s)", 303: "303 Redirection: See Other (%s)", 304: "304 Redirection: Not Modified (%s)", 305: "305 Redirection: Use Proxy (%s)", 306: "306 Redirection: (Unused) (%s)", 307: "307 Redirection: Temporary Redirect (%s)", 400: "400 Client Error: Bad Request (%s)", 401: "401 Client Error: Unauthorized (%s)", 402: "402 Client Error: Payment Required (%s)", 403: "403 Client Error: Forbidden (%s)", 404: "404 Client Error: Not Found (%s)", 405: "405 Client Error: Method Not Allowed (%s)", 406: "406 Client Error: Not Acceptable (%s)", 407: "407 Client Error: Proxy Authentication Required (%s)", 408: "408 Client Error: Request Timeout (%s)", 409: "409 Client Error: Conflict (%s)", 410: "410 Client Error: Gone (%s)", 411: "411 Client Error: Length Required (%s)", 412: "412 Client Error: Precondition Failed (%s)", 413: "413 Client Error: Request Entity Too Large (%s)", 414: "414 Client Error: Request-URI Too Long (%s)", 415: "415 Client Error: Unsupported Media Type (%s)", 416: "416 Client Error: Requested Range Not Satisfiable (%s)", 417: "417 Client Error: Expectation Failed (%s)", 500: "500 Server Error: Internal Server Error (%s)", 501: "501 Server Error: Not Implemented (%s)", 502: "502 Server Error: Bad Gateway (%s)", 503: "503 Server Error: Service Unavailable (%s)", 504: "504 Server Error: Gateway Timeout (%s)", 505: "505 Server Error: HTTP Version Not Supported (%s)",
ae89482001-08-13Anders Johansson  1000: "Disallowed by robots.txt. (%s)", 1001: "Can't handle scheme. (%s)",
898c182001-08-27Johan Schön  1002: "No matching filter. (%s)",
11b2ed2007-03-15Jonas Wallden  1003: "Too large content file -- indexing metadata only. (%s)",
898c182001-08-27Johan Schön  1100: "Failed to connect to %s.",
c88b0b2001-07-04Martin Nilsson  ]);
c253d42001-07-12Johan Schön 
c88b0b2001-07-04Martin Nilsson 
46a6522001-08-06Martin Nilsson  //!
7d85aa2001-07-04Martin Nilsson  array(array(string|int)) get_log( int profile, array(string) types, int from, int to ) {
c88b0b2001-07-04Martin Nilsson  string sql = "";
5990902001-07-05Martin Nilsson #define SQLADD(X) do{sizeof(sql)?(sql+=" AND "+(X)):(sql=" WHERE "+(X));}while(0)
c88b0b2001-07-04Martin Nilsson  if(profile) SQLADD("profile=" + profile); if(!sizeof(types)) return ({}); if(sizeof(types)!=3) SQLADD("(type='" + (types*"' OR type='") + "')"); if(from) SQLADD("at>" + from); if(to) SQLADD("to<" + to); #undef SQLADD Sql.Sql db = get_db();
7d85aa2001-07-04Martin Nilsson  if(!db) return ({});
c88b0b2001-07-04Martin Nilsson 
2cd10c2001-07-05Martin Nilsson  return map(db->query("SELECT unix_timestamp(at) as at,profile,code,type,extra FROM eventlog" + sql + " ORDER BY event DESC"),
c88b0b2001-07-04Martin Nilsson  lambda(mapping in) {
7d85aa2001-07-04Martin Nilsson  return ({ (int)in->at, (int)in->profile, in->type,
2cd10c2001-07-05Martin Nilsson  in->extra?sprintf(codes[(int)in->code], @(in->extra/"\n")): codes[(int)in->code] });
c88b0b2001-07-04Martin Nilsson  } ); } }