Roxen.git / server / base_server / roxen.pike

version» Context lines:

Roxen.git/server/base_server/roxen.pike:1:   // This file is part of Roxen WebServer.   // Copyright © 1996 - 2004, Roxen IS.   //   // The Roxen WebServer main program.   //   // Per Hedbor, Henrik Grubbström, Pontus Hagland, David Hedbor and others.   // ABS and suicide systems contributed freely by Francesco Chemolli    - constant cvs_version="$Id: roxen.pike,v 1.949 2006/11/16 14:19:43 mast Exp $"; + constant cvs_version="$Id: roxen.pike,v 1.950 2006/11/16 16:45:49 mast Exp $";      //! @appears roxen   //!   //! The Roxen WebServer main program.      // The argument cache. Used by the image cache.   ArgCache argcache;      // Some headerfiles   #define IN_ROXEN
Roxen.git/server/base_server/roxen.pike:3780: Inside #if 0 and #if defined(DEBUG)
  #if 0   #ifdef DEBUG    report_debug (describe_backtrace (err));   #endif   #endif    }    }   }       - #ifdef ENABLE_NEW_ARGCACHE +    class ArgCache   //! Generic cache for storing away a persistent mapping of data to be   //! refetched later by a short string key. This being a cache, your   //! data may be thrown away at random when the cache is full.   {   #undef QUERY   #define QUERY(X,Y...) db->query(X,Y)    Sql.Sql db;    string name;      #define CACHE_SIZE 900    - #ifdef THREADS +     Thread.Mutex mutex = Thread.Mutex();    // Allow recursive locks, since it's normal here.   # define LOCK() mixed __; catch( __ = mutex->lock() ) - #else - # define LOCK() - #endif +       #ifdef ARGCACHE_DEBUG   #define dwerror(ARGS...) werror(ARGS)   #else   #define dwerror(ARGS...) 0   #endif       static mapping(string|int:mixed) cache = ([ ]);       static void setup_table()
Roxen.git/server/base_server/roxen.pike:4086:    }       void refresh_arg(string id)    //! Indicate that the entry @[id] needs to be included in the next    //! @[write_dump]. @[id] must be an existing entry.    {    QUERY("UPDATE "+name+"2 SET rep_time=NOW() WHERE id = %s", id);    }   }    - #else // ENABLE_NEW_ARGCACHE -  - class ArgCache - //! Generic cache for storing away a persistent mapping of data to be - //! refetched later by a short string key. This being a cache, your - //! data may be thrown away at random when the cache is full. - { - #undef QUERY - #define QUERY(X,Y...) db->query(X,Y) -  Sql.Sql db; -  string name; -  - #define CACHE_VALUE 0 - #define CACHE_SKEY 1 - #define CACHE_SIZE 900 - #define CLEAN_SIZE 100 -  -  static string lq, ulq; -  - #ifdef THREADS -  Thread.Mutex mutex = Thread.Mutex(); -  // Allow recursive locks, since it's normal here. - # define LOCK() mixed __; catch( __ = mutex->lock() ) - #else - # define LOCK() - #endif -  -  static mapping(string|int:mixed) cache = ([ ]); -  -  static void setup_table() -  { -  if(catch(QUERY("SELECT md5 FROM "+name+" WHERE id=0"))) -  { -  master()->resolv("DBManager.is_module_table") -  ( 0, "local", name, -  "The argument cache, used to map between " -  "a short unique string and an argument " -  "mapping" ); -  catch(QUERY("DROP TABLE "+name )); -  QUERY("CREATE TABLE "+name+" (" -  "id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, " -  "index_id INT UNSIGNED NULL DEFAULT NULL, " -  "md5 CHAR(32) NOT NULL DEFAULT '', " -  "atime INT UNSIGNED NOT NULL DEFAULT 0, " -  "contents BLOB NOT NULL DEFAULT '', " -  "INDEX hind (md5))"); -  } -  // Add column index_id if it doesn't exists. -  if(catch(QUERY("SELECT index_id FROM "+name+" WHERE id=0"))) -  { -  QUERY("ALTER TABLE "+name+" " -  "ADD index_id INT UNSIGNED NULL DEFAULT NULL"); -  } -  } -  -  static void init_db() -  { -  // Delay DBManager resolving to before the 'roxen' object is -  // compiled. -  cache = ([]); -  db = dbm_cached_get("local"); -  setup_table( ); -  } -  -  static void create( string _name ) -  { -  name = _name; -  init_db(); -  // Support that the 'local' database moves (not really nessesary, -  // but it won't hurt either) -  master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db ); -  get_plugins(); -  } -  -  string read_args( int id ) -  { -  LOCK(); -  array res = QUERY("SELECT contents FROM "+name+" WHERE id="+id); -  if( sizeof(res) ) -  { -  QUERY("UPDATE "+name+" SET atime='"+time(1)+"' WHERE id="+id); -  return res[0]->contents; -  } -  return 0; -  } -  -  int read_index_id( int id ) -  { -  LOCK(); -  array res = QUERY("SELECT index_id FROM "+name+" WHERE id="+id); -  if(sizeof(res) && res[0]->index_id) -  return (int)res[0]->index_id; -  return -1; -  } -  -  int create_key( string long_key, string|void md, int|void index_id ) -  { -  if( !md ) md = md5(long_key); -  LOCK(); -  array data = -  QUERY("SELECT id,contents FROM "+name+" WHERE md5=%s", md ); -  -  foreach( data, mapping m ) -  if( m->contents == long_key ) -  return (int)m->id; -  -  if(zero_type(index_id)) -  index_id = -1; -  -  string index_id_value = (index_id == -1? "NULL": index_id); -  QUERY( "INSERT INTO "+name+" (contents,md5,atime,index_id) VALUES " -  "(" MYSQL__BINARY "%s,%s,UNIX_TIMESTAMP(),"+index_id_value+")", -  long_key, md ); -  int id = (int)db->master_sql->insert_id(); -  if(!id) -  error("ArgCache::create_key() insert_id returned 0.\n"); - #ifdef REPLICATE_DEBUG -  werror("Create new local key: id: %d, index_id: %d.\n", id, index_id); - #endif -  -  (plugins->create_key-({0}))( id, long_key ); -  -  return id; -  } -  -  static int low_key_exists( string key ) -  { -  LOCK(); -  int res = sizeof( QUERY( "SELECT id FROM "+name+" WHERE id="+(int)key)); -  // Fool optimizer. -  if( res ) -  return res; -  } -  -  string secret; -  -  static void ensure_secret() -  { -  if( !secret ) -  secret = query( "argcache_secret" ); -  } -  -  string encode_id( int a, int b, string|void server ) -  { -  ensure_secret(); -  object crypto = Crypto.arcfour(); -  crypto->set_encrypt_key( server||secret ); -  string res = crypto->crypt( a+"\327"+b ); -  // Ensure that we do not have a leading NUL. -  res[0] |= 0x80; -  res = Gmp.mpz( res, 256 )->digits( 36 ); -  return res; -  } -  -  static array plugins; -  static void get_plugins() -  { -  ensure_secret(); -  plugins = ({}); -  foreach( ({ "../local/arg_cache_plugins", "arg_cache_plugins" }), string d) -  if( file_stat( d ) ) -  foreach( glob("*.pike", get_dir( d )), string f ) -  { -  object plug = ((program)(d+"/"+f))(this_object()); -  if( !plug->disabled ) -  plugins += ({ plug }); -  } -  } -  -  static array plugin_decode_id( string id ) -  { -  mixed r; -  foreach( (plugins->decode_id-({0})), function(string:array(int)) f ) -  if( r = f( id ) ) -  return r; -  return 0; -  } -  -  array(int) low_decode_id(string a, string key) -  { -  if( catch( a = Gmp.mpz( a, 36 )->digits( 256 ) ) ) -  return 0; // Not very likely to work... -  object crypto = Crypto.arcfour(); -  crypto->set_encrypt_key(key); -  string msg = crypto->crypt(a); -  // Fix the high-order bit altered by encode_id(). -  msg[0] &= 0x7f; -  int i, j; -  if((sscanf(msg, "%d\327%d", i, j) == 2) && -  (msg == i + "\327" + j)) { -  return ({ i, j }); -  } - #ifndef NO_BROKEN_ARGCACHE_FALLBACK -  // Fallback -- Check if it's an old broken key. -  crypto->set_encrypt_key(key); -  msg = crypto->crypt("\0"+a); -  if((sscanf(msg, "%d\327%d", i, j) == 2) && -  (msg == i + "\327" + j)) { -  return ({ i, j }); -  } - #endif /* !NO_BROKEN_ARGCACHE_FALLBACK */ -  return 0; -  } -  -  static array(int) decode_id( string a ) -  { -  array(int) res; -  ensure_secret(); -  if (res = low_decode_id(a, secret)) { -  return res; -  } -  return plugin_decode_id(a); -  } -  -  int key_exists( string key ) -  //! Does the key 'key' exist in the cache? Returns 1 if it does, 0 -  //! if it was not present. -  { -  if( cache[key] ) return 1; -  array i = decode_id( key ); -  if(!i) return 0; -  return low_key_exists( i[0] ) && low_key_exists( i[1] ); -  } -  -  string store( mapping args ) -  //! Store a mapping (of purely encode_value:able data) in the -  //! argument cache. The string returned is your key to retrieve the -  //! data later. -  { -  array b = values(args), a = sort(indices(args),b); -  LOCK(); -  int index_id = low_store( a ); -  string id = encode_id( index_id, low_store( b, index_id ) ); -  if( !cache[ id ] ) -  cache[ id ] = args+([]); -  return id; -  } -  -  int low_store( array a, int|void index_id ) -  { -  string data = encode_value_canonic( a ); -  string hv = md5( data ); -  if( mixed q = cache[ hv ] ) -  return q; -  - #ifdef THREADS -  if( mixed q = cache[ hv ] ) -  return q; - #endif -  if( sizeof( cache ) >= CACHE_SIZE ) -  cache = ([]); -  -  int id = create_key( data, hv, index_id ); -  cache[ hv ] = id; -  cache[ id ] = a; -  return id; -  } -  -  mapping lookup( string id ) -  //! Recall a mapping stored in the cache. -  { -  if( cache[id] ) -  return cache[id]+([]); -  array i = decode_id( id ); -  if( !i ) -  error("Requesting unknown key (decode failed)\n"); -  array a = low_lookup( i[0] ); -  array b = low_lookup( i[1] ); -  if (!arrayp (a) || !arrayp (b) || sizeof (a) != sizeof (b)) -  { -  // Got lookup with ids from an old cache which has been zapped, -  // and the entries are now used for something else. - #ifdef ARG_CACHE_DEBUG -  werror("lookup(%O) a: %O, b: %O\n", id, a, b); - #endif -  error("Requesting unknown key (size missmatch)\n"); -  } -  return (cache[id] = mkmapping( a, b ))+([]); -  } -  -  array low_lookup( int id ) -  { -  mixed v; -  if( v = cache[id] ) -  return v; -  string q = read_args( id ); -  if( !q ) -  error("Requesting unknown key (not found in db)\n"); -  mixed data = decode_value(q); -  string hl = Crypto.md5()->update( q )->digest(); -  cache[ hl ] = id; -  cache[ id ] = data; -  return data; -  } -  -  void delete( string id ) -  //! Remove the data element stored under the key 'id'. -  { -  LOCK(); -  (plugins->delete-({0}))( id ); -  m_delete( cache, id ); -  -  foreach( decode_id( id ), int id ) -  { -  (plugins->low_delete-({0}))( id ); -  if(cache[id]) -  { -  m_delete( cache, cache[id] ); -  m_delete( cache, id ); -  } -  QUERY( "DELETE FROM "+name+" WHERE id="+id ); -  } -  } -  - #define SECRET_TAG "££" -  -  int write_dump(Stdio.File file, int|void from_time) -  // Write a mapping from id to encoded arg string for all local arg -  // entries created after from_time to a file. Returns 0 if faled, 1 -  // otherwise. -  { -  constant FETCH_ROWS = 10000; -  -  string encoded_secret = SECRET_TAG+MIME.encode_base64(secret, 1)+"\n"; -  if(sizeof(encoded_secret) != file->write(encoded_secret)) -  return 0; -  -  // The server does only need to use file based argcache -  // replication if the server don't participate in a replicate -  // setup with a shared database. -  if( !has_value((plugins->is_functional-({0}))(), 1) ) -  { -  int cursor; -  array(int) ids; -  do { -  if(from_time) -  // Only replicate entries accessed during the prefetch crawling. -  ids = -  (array(int)) -  QUERY( "SELECT id from "+name+ -  " WHERE atime >= %d " -  " AND index_id IS NOT NULL" -  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS)->id; -  else -  // Make sure _every_ entry is replicated when a dump is created. -  ids = -  (array(int)) -  QUERY( "SELECT id from "+name+ -  " LIMIT %d, %d", cursor, FETCH_ROWS)->id; -  -  cursor += FETCH_ROWS; -  -  foreach(ids, int id) { -  int index_id = read_index_id(id); - #ifdef REPLICATE_DEBUG -  werror("write_dump: argcache id: %d, index_id: %d.\n", id, index_id); - #endif -  -  string s = -  MIME.encode_base64(encode_value(({ id, read_args(id), -  index_id, read_args(index_id) })), -  1)+"\n"; -  if(sizeof(s) != file->write(s)) -  return 0; -  } -  } while(sizeof(ids) == FETCH_ROWS); -  } -  return file->write("EOF\n") == 4; -  } -  -  static void create_remote_key(int id, string key, -  int index_id, string index_key, -  string server) -  { -  (plugins->create_remote_key-({0}))( id, key, index_id, index_key, server ); -  } -  -  string read_dump (Stdio.FILE file) -  // Returns an error message if there was a parse error, 0 otherwise. -  { -  string secret = file->gets(); -  // Note, old replication dumps can contain unencoded server secrets. -  if(secret && has_prefix(secret, SECRET_TAG)) -  secret = MIME.decode_base64(secret[sizeof(SECRET_TAG)..]); -  -  if(!secret || !sizeof(secret)) -  return "Server secret is missing\n"; -  -  string s; -  while(s = file->gets()) -  { -  if(s == "EOF") -  return 0; -  array a; -  if(catch { -  a = decode_value(MIME.decode_base64(s)); -  }) return "Decode failed for argcache record\n"; -  -  if(sizeof(a) != 4) -  return "Decode failed for argcache record (wrong size on key array)\n"; -  - #ifdef REPLICATE_DEBUG -  werror("read_dump: argcache id: %d, index_id: %d.\n", a[0], a[2]); - #endif -  create_remote_key(a[0], a[1], a[2], a[3], secret); -  } -  if(s != "EOF") -  return "Missing data in argcache file\n"; -  return 0; -  } -  -  void refresh_arg(string id) -  { -  array i = decode_id( id ); -  if( !i ) -  error("Requesting unknown key (decode failed)\n"); -  LOCK(); -  QUERY("UPDATE "+name+" SET atime='"+time(1)+"' WHERE id="+i[0]); -  QUERY("UPDATE "+name+" SET atime='"+time(1)+"' WHERE id="+i[1]); -  } - } - #endif // ENABLE_NEW_ARGCACHE -  +    mapping cached_decoders = ([]);   string decode_charset( string charset, string data )   {    // FIXME: This code is probably not thread-safe!    if( charset == "iso-8859-1" ) return data;    if( !cached_decoders[ charset ] )    cached_decoders[ charset ] = Locale.Charset.decoder( charset );    data = cached_decoders[ charset ]->feed( data )->drain();    cached_decoders[ charset ]->clear();    return data;