Roxen.git/ server/ base_server/ roxen.pike

Branch: Tag:

2006-11-16

2006-11-16 16:45:50 by Martin Stjernholm <mast@lysator.liu.se>

Removed the old argcache system so that the new one is used by default (i.e.
ENABLE_NEW_ARGCACHE is no longer necessary).

Upgrade note 1: There is no compatibility fallback to read argcache entries
from the old database. An upgrade instead depends on that the image cache
remains intact for long enough so that old images can be served without
requiring their argcache entries.

Upgrade note 2: The old 'arguments' table in the local database is not
dropped automatically. Administrators are adviced to do that to free up
space.

Rev: server/base_server/roxen.pike:1.950

6:   // Per Hedbor, Henrik Grubbström, Pontus Hagland, David Hedbor and others.   // ABS and suicide systems contributed freely by Francesco Chemolli    - constant cvs_version="$Id: roxen.pike,v 1.949 2006/11/16 14:19:43 mast Exp $"; + constant cvs_version="$Id: roxen.pike,v 1.950 2006/11/16 16:45:49 mast Exp $";      //! @appears roxen   //!
3787:   }       - #ifdef ENABLE_NEW_ARGCACHE +    class ArgCache   //! Generic cache for storing away a persistent mapping of data to be   //! refetched later by a short string key. This being a cache, your
3800:      #define CACHE_SIZE 900    - #ifdef THREADS +     Thread.Mutex mutex = Thread.Mutex();    // Allow recursive locks, since it's normal here.   # define LOCK() mixed __; catch( __ = mutex->lock() ) - #else - # define LOCK() - #endif +       #ifdef ARGCACHE_DEBUG   #define dwerror(ARGS...) werror(ARGS)
4093:    }   }    - #else // ENABLE_NEW_ARGCACHE -  - class ArgCache - //! Generic cache for storing away a persistent mapping of data to be - //! refetched later by a short string key. This being a cache, your - //! data may be thrown away at random when the cache is full. - { - #undef QUERY - #define QUERY(X,Y...) db->query(X,Y) -  Sql.Sql db; -  string name; -  - #define CACHE_VALUE 0 - #define CACHE_SKEY 1 - #define CACHE_SIZE 900 - #define CLEAN_SIZE 100 -  -  static string lq, ulq; -  - #ifdef THREADS -  Thread.Mutex mutex = Thread.Mutex(); -  // Allow recursive locks, since it's normal here. - # define LOCK() mixed __; catch( __ = mutex->lock() ) - #else - # define LOCK() - #endif -  -  static mapping(string|int:mixed) cache = ([ ]); -  -  static void setup_table() -  { -  if(catch(QUERY("SELECT md5 FROM "+name+" WHERE id=0"))) -  { -  master()->resolv("DBManager.is_module_table") -  ( 0, "local", name, -  "The argument cache, used to map between " -  "a short unique string and an argument " -  "mapping" ); -  catch(QUERY("DROP TABLE "+name )); -  QUERY("CREATE TABLE "+name+" (" -  "id INT UNSIGNED AUTO_INCREMENT PRIMARY KEY, " -  "index_id INT UNSIGNED NULL DEFAULT NULL, " -  "md5 CHAR(32) NOT NULL DEFAULT '', " -  "atime INT UNSIGNED NOT NULL DEFAULT 0, " -  "contents BLOB NOT NULL DEFAULT '', " -  "INDEX hind (md5))"); -  } -  // Add column index_id if it doesn't exists. -  if(catch(QUERY("SELECT index_id FROM "+name+" WHERE id=0"))) -  { -  QUERY("ALTER TABLE "+name+" " -  "ADD index_id INT UNSIGNED NULL DEFAULT NULL"); -  } -  } -  -  static void init_db() -  { -  // Delay DBManager resolving to before the 'roxen' object is -  // compiled. -  cache = ([]); -  db = dbm_cached_get("local"); -  setup_table( ); -  } -  -  static void create( string _name ) -  { -  name = _name; -  init_db(); -  // Support that the 'local' database moves (not really nessesary, -  // but it won't hurt either) -  master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db ); -  get_plugins(); -  } -  -  string read_args( int id ) -  { -  LOCK(); -  array res = QUERY("SELECT contents FROM "+name+" WHERE id="+id); -  if( sizeof(res) ) -  { -  QUERY("UPDATE "+name+" SET atime='"+time(1)+"' WHERE id="+id); -  return res[0]->contents; -  } -  return 0; -  } -  -  int read_index_id( int id ) -  { -  LOCK(); -  array res = QUERY("SELECT index_id FROM "+name+" WHERE id="+id); -  if(sizeof(res) && res[0]->index_id) -  return (int)res[0]->index_id; -  return -1; -  } -  -  int create_key( string long_key, string|void md, int|void index_id ) -  { -  if( !md ) md = md5(long_key); -  LOCK(); -  array data = -  QUERY("SELECT id,contents FROM "+name+" WHERE md5=%s", md ); -  -  foreach( data, mapping m ) -  if( m->contents == long_key ) -  return (int)m->id; -  -  if(zero_type(index_id)) -  index_id = -1; -  -  string index_id_value = (index_id == -1? "NULL": index_id); -  QUERY( "INSERT INTO "+name+" (contents,md5,atime,index_id) VALUES " -  "(" MYSQL__BINARY "%s,%s,UNIX_TIMESTAMP(),"+index_id_value+")", -  long_key, md ); -  int id = (int)db->master_sql->insert_id(); -  if(!id) -  error("ArgCache::create_key() insert_id returned 0.\n"); - #ifdef REPLICATE_DEBUG -  werror("Create new local key: id: %d, index_id: %d.\n", id, index_id); - #endif -  -  (plugins->create_key-({0}))( id, long_key ); -  -  return id; -  } -  -  static int low_key_exists( string key ) -  { -  LOCK(); -  int res = sizeof( QUERY( "SELECT id FROM "+name+" WHERE id="+(int)key)); -  // Fool optimizer. -  if( res ) -  return res; -  } -  -  string secret; -  -  static void ensure_secret() -  { -  if( !secret ) -  secret = query( "argcache_secret" ); -  } -  -  string encode_id( int a, int b, string|void server ) -  { -  ensure_secret(); -  object crypto = Crypto.arcfour(); -  crypto->set_encrypt_key( server||secret ); -  string res = crypto->crypt( a+"\327"+b ); -  // Ensure that we do not have a leading NUL. -  res[0] |= 0x80; -  res = Gmp.mpz( res, 256 )->digits( 36 ); -  return res; -  } -  -  static array plugins; -  static void get_plugins() -  { -  ensure_secret(); -  plugins = ({}); -  foreach( ({ "../local/arg_cache_plugins", "arg_cache_plugins" }), string d) -  if( file_stat( d ) ) -  foreach( glob("*.pike", get_dir( d )), string f ) -  { -  object plug = ((program)(d+"/"+f))(this_object()); -  if( !plug->disabled ) -  plugins += ({ plug }); -  } -  } -  -  static array plugin_decode_id( string id ) -  { -  mixed r; -  foreach( (plugins->decode_id-({0})), function(string:array(int)) f ) -  if( r = f( id ) ) -  return r; -  return 0; -  } -  -  array(int) low_decode_id(string a, string key) -  { -  if( catch( a = Gmp.mpz( a, 36 )->digits( 256 ) ) ) -  return 0; // Not very likely to work... -  object crypto = Crypto.arcfour(); -  crypto->set_encrypt_key(key); -  string msg = crypto->crypt(a); -  // Fix the high-order bit altered by encode_id(). -  msg[0] &= 0x7f; -  int i, j; -  if((sscanf(msg, "%d\327%d", i, j) == 2) && -  (msg == i + "\327" + j)) { -  return ({ i, j }); -  } - #ifndef NO_BROKEN_ARGCACHE_FALLBACK -  // Fallback -- Check if it's an old broken key. -  crypto->set_encrypt_key(key); -  msg = crypto->crypt("\0"+a); -  if((sscanf(msg, "%d\327%d", i, j) == 2) && -  (msg == i + "\327" + j)) { -  return ({ i, j }); -  } - #endif /* !NO_BROKEN_ARGCACHE_FALLBACK */ -  return 0; -  } -  -  static array(int) decode_id( string a ) -  { -  array(int) res; -  ensure_secret(); -  if (res = low_decode_id(a, secret)) { -  return res; -  } -  return plugin_decode_id(a); -  } -  -  int key_exists( string key ) -  //! Does the key 'key' exist in the cache? Returns 1 if it does, 0 -  //! if it was not present. -  { -  if( cache[key] ) return 1; -  array i = decode_id( key ); -  if(!i) return 0; -  return low_key_exists( i[0] ) && low_key_exists( i[1] ); -  } -  -  string store( mapping args ) -  //! Store a mapping (of purely encode_value:able data) in the -  //! argument cache. The string returned is your key to retrieve the -  //! data later. -  { -  array b = values(args), a = sort(indices(args),b); -  LOCK(); -  int index_id = low_store( a ); -  string id = encode_id( index_id, low_store( b, index_id ) ); -  if( !cache[ id ] ) -  cache[ id ] = args+([]); -  return id; -  } -  -  int low_store( array a, int|void index_id ) -  { -  string data = encode_value_canonic( a ); -  string hv = md5( data ); -  if( mixed q = cache[ hv ] ) -  return q; -  - #ifdef THREADS -  if( mixed q = cache[ hv ] ) -  return q; - #endif -  if( sizeof( cache ) >= CACHE_SIZE ) -  cache = ([]); -  -  int id = create_key( data, hv, index_id ); -  cache[ hv ] = id; -  cache[ id ] = a; -  return id; -  } -  -  mapping lookup( string id ) -  //! Recall a mapping stored in the cache. -  { -  if( cache[id] ) -  return cache[id]+([]); -  array i = decode_id( id ); -  if( !i ) -  error("Requesting unknown key (decode failed)\n"); -  array a = low_lookup( i[0] ); -  array b = low_lookup( i[1] ); -  if (!arrayp (a) || !arrayp (b) || sizeof (a) != sizeof (b)) -  { -  // Got lookup with ids from an old cache which has been zapped, -  // and the entries are now used for something else. - #ifdef ARG_CACHE_DEBUG -  werror("lookup(%O) a: %O, b: %O\n", id, a, b); - #endif -  error("Requesting unknown key (size missmatch)\n"); -  } -  return (cache[id] = mkmapping( a, b ))+([]); -  } -  -  array low_lookup( int id ) -  { -  mixed v; -  if( v = cache[id] ) -  return v; -  string q = read_args( id ); -  if( !q ) -  error("Requesting unknown key (not found in db)\n"); -  mixed data = decode_value(q); -  string hl = Crypto.md5()->update( q )->digest(); -  cache[ hl ] = id; -  cache[ id ] = data; -  return data; -  } -  -  void delete( string id ) -  //! Remove the data element stored under the key 'id'. -  { -  LOCK(); -  (plugins->delete-({0}))( id ); -  m_delete( cache, id ); -  -  foreach( decode_id( id ), int id ) -  { -  (plugins->low_delete-({0}))( id ); -  if(cache[id]) -  { -  m_delete( cache, cache[id] ); -  m_delete( cache, id ); -  } -  QUERY( "DELETE FROM "+name+" WHERE id="+id ); -  } -  } -  - #define SECRET_TAG "££" -  -  int write_dump(Stdio.File file, int|void from_time) -  // Write a mapping from id to encoded arg string for all local arg -  // entries created after from_time to a file. Returns 0 if faled, 1 -  // otherwise. -  { -  constant FETCH_ROWS = 10000; -  -  string encoded_secret = SECRET_TAG+MIME.encode_base64(secret, 1)+"\n"; -  if(sizeof(encoded_secret) != file->write(encoded_secret)) -  return 0; -  -  // The server does only need to use file based argcache -  // replication if the server don't participate in a replicate -  // setup with a shared database. -  if( !has_value((plugins->is_functional-({0}))(), 1) ) -  { -  int cursor; -  array(int) ids; -  do { -  if(from_time) -  // Only replicate entries accessed during the prefetch crawling. -  ids = -  (array(int)) -  QUERY( "SELECT id from "+name+ -  " WHERE atime >= %d " -  " AND index_id IS NOT NULL" -  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS)->id; -  else -  // Make sure _every_ entry is replicated when a dump is created. -  ids = -  (array(int)) -  QUERY( "SELECT id from "+name+ -  " LIMIT %d, %d", cursor, FETCH_ROWS)->id; -  -  cursor += FETCH_ROWS; -  -  foreach(ids, int id) { -  int index_id = read_index_id(id); - #ifdef REPLICATE_DEBUG -  werror("write_dump: argcache id: %d, index_id: %d.\n", id, index_id); - #endif -  -  string s = -  MIME.encode_base64(encode_value(({ id, read_args(id), -  index_id, read_args(index_id) })), -  1)+"\n"; -  if(sizeof(s) != file->write(s)) -  return 0; -  } -  } while(sizeof(ids) == FETCH_ROWS); -  } -  return file->write("EOF\n") == 4; -  } -  -  static void create_remote_key(int id, string key, -  int index_id, string index_key, -  string server) -  { -  (plugins->create_remote_key-({0}))( id, key, index_id, index_key, server ); -  } -  -  string read_dump (Stdio.FILE file) -  // Returns an error message if there was a parse error, 0 otherwise. -  { -  string secret = file->gets(); -  // Note, old replication dumps can contain unencoded server secrets. -  if(secret && has_prefix(secret, SECRET_TAG)) -  secret = MIME.decode_base64(secret[sizeof(SECRET_TAG)..]); -  -  if(!secret || !sizeof(secret)) -  return "Server secret is missing\n"; -  -  string s; -  while(s = file->gets()) -  { -  if(s == "EOF") -  return 0; -  array a; -  if(catch { -  a = decode_value(MIME.decode_base64(s)); -  }) return "Decode failed for argcache record\n"; -  -  if(sizeof(a) != 4) -  return "Decode failed for argcache record (wrong size on key array)\n"; -  - #ifdef REPLICATE_DEBUG -  werror("read_dump: argcache id: %d, index_id: %d.\n", a[0], a[2]); - #endif -  create_remote_key(a[0], a[1], a[2], a[3], secret); -  } -  if(s != "EOF") -  return "Missing data in argcache file\n"; -  return 0; -  } -  -  void refresh_arg(string id) -  { -  array i = decode_id( id ); -  if( !i ) -  error("Requesting unknown key (decode failed)\n"); -  LOCK(); -  QUERY("UPDATE "+name+" SET atime='"+time(1)+"' WHERE id="+i[0]); -  QUERY("UPDATE "+name+" SET atime='"+time(1)+"' WHERE id="+i[1]); -  } - } - #endif // ENABLE_NEW_ARGCACHE -  +    mapping cached_decoders = ([]);   string decode_charset( string charset, string data )   {