Roxen.git / server / base_server / roxen.pike

version» Context lines:

Roxen.git/server/base_server/roxen.pike:1:   // This file is part of Roxen WebServer.   // Copyright © 1996 - 2009, Roxen IS.   //   // The Roxen WebServer main program.   //   // Per Hedbor, Henrik Grubbström, Pontus Hagland, David Hedbor and others.   // ABS and suicide systems contributed freely by Francesco Chemolli    - constant cvs_version="$Id: roxen.pike,v 1.1044 2009/11/18 13:48:58 mast Exp $"; + constant cvs_version="$Id: roxen.pike,v 1.1045 2009/11/23 15:38:41 grubba Exp $";      //! @appears roxen   //!   //! The Roxen WebServer main program.      // The argument cache. Used by the image cache.   ArgCache argcache;      // Some headerfiles   #define IN_ROXEN
Roxen.git/server/base_server/roxen.pike:4205:    master()->resolv("DBManager.is_module_table")    ( 0, "local", name+"2",    "The argument cache, used to map between "    "a unique string and an argument mapping" );    catch(QUERY("DROP TABLE "+name+"2" ));    QUERY("CREATE TABLE "+name+"2 ("    "id CHAR(32) PRIMARY KEY, "    "ctime DATETIME NOT NULL, "    "atime DATETIME NOT NULL, "    "rep_time DATETIME NOT NULL, " +  "timeout INT NULL, "    "contents MEDIUMBLOB NOT NULL)"); -  +  // FIXME: Add index on timeout for garb?    }       if (catch (QUERY ("SELECT rep_time FROM " + name + "2 LIMIT 0")))    {    // Upgrade a table without rep_time.    QUERY ("ALTER TABLE " + name + "2"    " ADD rep_time DATETIME NOT NULL"    " AFTER atime");    }    -  +  if (catch (QUERY ("SELECT timeout FROM " + name + "2 LIMIT 0"))) +  { +  // Upgrade a table without timeout. +  QUERY ("ALTER TABLE " + name + "2" +  " ADD timeout INT NULL" +  " AFTER rep_time"); +  } +     catch {    array(mapping(string:mixed)) res =    QUERY("DESCRIBE "+name+"2 contents");       if(res[0]->Type == "blob") {    QUERY("ALTER TABLE "+name+"2 MODIFY contents MEDIUMBLOB NOT NULL");    werror("ArgCache: Extending \"contents\" field in table \"%s2\" from BLOB to MEDIUMBLOB.\n", name);    }    };    }    -  +  protected void do_cleanup() +  { +  QUERY("DELETE LOW_PRIORITY FROM " + name + "2 " +  " WHERE timeout IS NOT NULL " +  " AND timeout < %d", time()); +  } +  +  protected void cleanup_process( ) +  { +  // Flushes may be costly in large sites (since there's no index +  // on the timeout field) so schedule next run sometime after +  // 04:30 the day after tomorrow. +  int now = time(); +  mapping info = localtime(now); +  int wait = (int) ((24 - info->hour) + 24 + 4.5) * 3600 + random(500); +  background_run(wait, cleanup_process); +  +  do_cleanup(); +  } +     protected void init_db()    {    // Delay DBManager resolving to before the 'roxen' object is    // compiled.    cache = ([]);    db = dbm_cached_get("local");    setup_table( ); -  +  +  // Cleanup exprired entries on start. +  background_run( 10, cleanup_process );    }       protected void create( string _name )    {    name = _name;    init_db();    // Support that the 'local' database moves (not really nessesary,    // but it won't hurt either)    master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db );    get_plugins();    }       protected string read_encoded_args( string id, int dont_update_atime )    {    LOCK();    array res = QUERY("SELECT contents FROM "+name+"2 "    " WHERE id = %s", id);    if(!sizeof(res))    return 0;    if (!dont_update_atime) -  QUERY("UPDATE "+name+"2 " +  QUERY("UPDATE LOW_PRIORITY "+name+"2 "    " SET atime = NOW() "    " WHERE id = %s", id);    return res[0]->contents;    }       // Callback used in replicate.pike -  void create_key( string id, string encoded_args ) +  void create_key( string id, string encoded_args, int|void timeout )    { -  +  if (!zero_type(timeout) && (timeout < time(1))) return; // Expired.    LOCK();    array(mapping) rows =    QUERY("SELECT id, contents FROM "+name+"2 WHERE id = %s", id );    foreach( rows, mapping row )    if( row->contents != encoded_args ) {    report_error("ArgCache.create_key(): "    "Duplicate key found! Please report this to support@roxen.com: "    "id: %O, old data: %O, new data: %O\n",    id, row->contents, encoded_args);    error("ArgCache.create_key() Duplicate key found!\n");    }       if(sizeof(rows)) { -  QUERY("UPDATE "+name+"2 " +  QUERY("UPDATE LOW_PRIORITY "+name+"2 "    " SET atime = NOW() "    " WHERE id = %s", id); -  +  if (zero_type(timeout)) { +  QUERY("UPDATE LOW_PRIORITY "+name+"2 " +  " SET timeout = NULL " +  " WHERE id = %s", id); +  } else { +  QUERY("UPDATE LOW_PRIORITY "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s " +  " AND timeout IS NOT NULL " +  " AND timeout < %d", +  timeout, id, timeout); +  }    return;    }       QUERY( "INSERT INTO "+name+"2 "    "(id, contents, ctime, atime) VALUES "    "(%s, " MYSQL__BINARY "%s, NOW(), NOW())", id, encoded_args );    -  +  if (!zero_type(timeout)) { +  QUERY("UPDATE LOW_PRIORITY "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s", +  timeout, id); +  } +     dwerror("ArgCache: Create new key %O\n", id);       (plugins->create_key-({0}))( id, encoded_args );    }       protected array plugins;    protected void get_plugins()    {    plugins = ({});    foreach( ({ "../local/arg_cache_plugins", "arg_cache_plugins" }), string d)
Roxen.git/server/base_server/roxen.pike:4314:       protected mapping plugins_read_encoded_args( string id )    {    mapping args;    foreach( (plugins->read_encoded_args - ({0})), function(string:mapping) f )    if( args = f( id ) )    return args;    return 0;    }    -  string store( mapping args ) +  string store( mapping args, int|void timeout )    //! Store a mapping (of purely encode_value:able data) in the    //! argument cache. The string returned is your key to retrieve the    //! data later. -  +  //! +  //! @param timeout +  //! Timeout for the entry in seconds from now. If @expr{UNDEFINED@}, +  //! the entry will not expire.    { -  +  args += ([]); +  if (!zero_type(timeout)) timeout += time();    string encoded_args = encode_value_canonic( args );    string id = Gmp.mpz(Crypto.SHA1.hash(encoded_args), 256)->digits(36); -  if( cache[ id ] ) +  if( cache[ id ] ) { +  if (zero_type(timeout)) { +  // No timeout. +  QUERY("UPDATE LOW_PRIORITY "+name+"2 " +  " SET timeout = NULL " +  " WHERE id = %s", id); +  } else { +  QUERY("UPDATE LOW_PRIORITY "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s " +  " AND timeout IS NOT NULL " +  " AND timeout < %d", +  timeout, id, timeout); +  }    return id; -  create_key(id, encoded_args); -  if( !cache[ id ] ) -  cache[ id ] = args+([]); +  } +  create_key(id, encoded_args, timeout);    if( sizeof( cache ) >= CACHE_SIZE )    cache = ([]); -  +  if( !cache[ id ] ) +  cache[ id ] = args;    return id;    }          mapping lookup( string id )    //! Recall a mapping stored in the cache.    {    if( cache[id] )    return cache[id] + ([]);    string encoded_args = (read_encoded_args(id, 0) ||    plugins_read_encoded_args(id));    if(!encoded_args) {    error("Requesting unknown key (not found in db)\n");    }    mapping args = decode_value(encoded_args); -  cache[id] = args + ([]); +     if( sizeof( cache ) >= CACHE_SIZE )    // Yowza! Garbing bulldoze style. /mast    cache = ([]); -  +  cache[id] = args + ([]);    return args;    }       void delete( string id )    //! Remove the data element stored under the key @[id].    {    LOCK();    (plugins->delete-({0}))( id );    m_delete( cache, id );   
Roxen.git/server/base_server/roxen.pike:4403:    array(string) ids;    do {    // Note: No lock is held, so rows might be added between the    // SELECTs here. That can however only cause a slight overlap    // between the LIMIT windows since rows are only added and    // never removed, and read_dump doesn't mind the occasional    // duplicate entry.    //    // A lock will be necessary here if a garb is added, though.    +  array(mapping(string:string)) entries;    if(from_time)    // Only replicate entries accessed during the prefetch crawling. -  ids = -  (array(string)) -  QUERY( "SELECT id from "+name+"2 " +  entries = +  QUERY( "SELECT id, timeout from "+name+"2 "    " WHERE rep_time >= FROM_UNIXTIME(%d) " -  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS)->id; +  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS);    else    // Make sure _every_ entry is replicated when a dump is created. -  ids = -  (array(string)) -  QUERY( "SELECT id from "+name+"2 " -  " LIMIT %d, %d", cursor, FETCH_ROWS)->id; +  entries = +  QUERY( "SELECT id, timeout from "+name+"2 " +  " LIMIT %d, %d", cursor, FETCH_ROWS);    -  +  ids = entries->id; +  array(string) timeouts = entries->timeout;    cursor += FETCH_ROWS;    -  foreach(ids, string id) { +  foreach(ids; int i; string id) {    dwerror("ArgCache.write_dump(): %O\n", id);       string encoded_args;    if (mapping args = cache[id])    encoded_args = encode_value_canonic (args);    else {    encoded_args = read_encoded_args (id, 1);    if (!encoded_args) error ("ArgCache entry %O disappeared.\n", id);    }    -  string s = +  string s; +  if (timeouts[i]) { +  int timeout = (int)timeouts[i]; +  if (timeout < time(1)) { +  // Expired entry. Don't replicate. +  continue; +  } +  s = +  MIME.encode_base64(encode_value(({ id, encoded_args, timeout })), +  1)+"\n"; +  } else { +  // No timeout. Backward-compatible format. +  s =    MIME.encode_base64(encode_value(({ id, encoded_args })),    1)+"\n"; -  +  }    if(sizeof(s) != file->write(s))    return 0;    entry_count++;    }    } while(sizeof(ids) == FETCH_ROWS);    }    if (file->write("EOF\n") != 4)    return 0;    return entry_count ? 1 : -1;    }
Roxen.git/server/base_server/roxen.pike:4478:    // index_id when from_time was zero, so we ignore them here    // instead.    dwerror ("ArgCache.read_dump(): entry ignored.\n");    else {    array v = decode_value(a[1]), i = decode_value(a[3]);   #if 0    dwerror ("ArgCache.read_dump(): values: %O, indices: %O\n", v, i);   #endif    store(mkmapping(i, v));    } -  } else if (sizeof(a) == 2) { -  // New style argcache dump. +  } else if ((sizeof(a) == 2) || (sizeof(a) == 3)) { +  // New style argcache dump, possibly with timeout.    dwerror("ArgCache.read_dump(): %O\n", a[0]); -  create_key(a[0], a[1]); +  create_key(@a);    } else    return "Decode failed for argcache record (wrong size on key array)\n";    }    if(s != "EOF")    return "Missing data in argcache file\n";    return 0;    }       void refresh_arg(string id)    //! Indicate that the entry @[id] needs to be included in the next