Roxen.git / server / base_server / roxen.pike

version» Context lines:

Roxen.git/server/base_server/roxen.pike:1:   // This file is part of Roxen WebServer.   // Copyright © 1996 - 2004, Roxen IS.   //   // The Roxen WebServer main program.   //   // Per Hedbor, Henrik Grubbström, Pontus Hagland, David Hedbor and others.   // ABS and suicide systems contributed freely by Francesco Chemolli    - constant cvs_version="$Id: roxen.pike,v 1.965 2009/06/29 13:22:54 mast Exp $"; + constant cvs_version="$Id: roxen.pike,v 1.966 2009/11/26 15:36:52 grubba Exp $";      //! @appears roxen   //!   //! The Roxen WebServer main program.      // The argument cache. Used by the image cache.   ArgCache argcache;      // Some headerfiles   #define IN_ROXEN
Roxen.git/server/base_server/roxen.pike:2873: Inside #if defined(ARG_CACHE_DEBUG)
   {   #ifdef ARG_CACHE_DEBUG    werror("draw %O\n", name );   #endif    mixed args = Array.map( Array.map( name/"$", argcache->lookup,    id->client ), frommapp);       mapping meta;    string data;    array guides; + #ifdef ARG_CACHE_DEBUG +  werror("draw args: %O\n", args ); + #endif    mixed reply = draw_function( @copy_value(args), id );       if( !reply ) {   #ifdef ARG_CACHE_DEBUG    werror("%O(%{%O, %}%O) ==> 0\n",    draw_function, args, id);   #endif    return;    }   
Roxen.git/server/base_server/roxen.pike:3622:    //!    //! A non-zero @[nodraw] parameter means an image not already in the    //! cache will not be rendered on the fly, but instead return zero.    {    mapping res = http_file_answer( args, id, nodraw );    return res && res->data;    }       mapping http_file_answer( array|string|mapping data,    RequestID id, -  int|void nodraw ) +  int|void nodraw, int|void timeout )    //! Returns a @[result mapping] like one generated by    //! @[Roxen.http_file_answer()] but for the image file    //! rendered from the `data' instructions.    //!    //! Like @[metadata], a non-zero @[nodraw]parameter means an    //! image not already in the cache will not be rendered on the fly,    //! but instead zero will be returned (this will be seen as a 'File    //! not found' error)    {    current_configuration->set(id->conf); -  string na = store( data,id ); +  string na = store( data, id, timeout );    mixed res;   #ifdef ARG_CACHE_DEBUG -  werror("data %O\n", na ); +  werror("data: %O id: %O\n", na, id );   #endif    if(! (res=restore( na,id )) )    {    mixed err;    if (nodraw || (err = catch {    if (mapping res = draw( na, id ))    return res;    })) {    // File not found.   
Roxen.git/server/base_server/roxen.pike:3722:    return res;    }       mapping tomapp( mixed what )    {    if( mappingp( what ))    return what;    return ([ "":what ]);    }    -  string store( array|string|mapping data, RequestID id ) +  string store( array|string|mapping data, RequestID id, int|void timeout )    //! Store the data your draw callback expects to receive as its    //! first argument(s). If the data is an array, the draw callback    //! will be called like <pi>callback( @@data, id )</pi>. -  +  //! +  //! @param timeout +  //! Timeout for the entry in seconds from now. If @expr{UNDEFINED@}, +  //! the entry will not expire. Currently just passed along to +  //! the @[ArgCache].    {    string ci, user;    function update_args = lambda ( mapping a )    {    if (!a->format)    // Make implicit format choice explicit   #if constant(Image.GIF) && constant(Image.GIF.encode)    a->format = "gif";   #else    a->format = "png";
Roxen.git/server/base_server/roxen.pike:3747:    if( id->misc->authenticated_user &&    !id->misc->authenticated_user->is_transient )    // This entry is not actually used, it's only there to    // generate a unique key.    a["\0u"] = user = id->misc->authenticated_user->name();    };       if( mappingp( data ) )    {    update_args( data ); -  ci = argcache->store( data ); +  ci = argcache->store( data, timeout );    }    else if( arrayp( data ) )    {    if( !mappingp( data[0] ) )    error("Expected mapping as the first element of the argument array\n");    update_args( data[0] ); -  ci = map( map( data, tomapp ), argcache->store )*"$"; +  ci = map( map( data, tomapp ), argcache->store, timeout )*"$";    } else    ci = data;    update_args = 0; // To avoid garbage.       if( zero_type( uid_cache[ ci ] ) )    {    uid_cache[ci] = user;    if( catch(QUERY("INSERT INTO "+name+" "    "(id,uid,atime) VALUES (%s,%s,UNIX_TIMESTAMP())",    ci, user||"")) )
Roxen.git/server/base_server/roxen.pike:3904:   #else   # define LOCK()   #endif      #ifdef ARGCACHE_DEBUG   #define dwerror(ARGS...) werror(ARGS)   #else   #define dwerror(ARGS...) 0   #endif    +  //! Cache of the latest entries requested or stored. +  //! Limited to @[CACHE_SIZE] (currently @expr{900@}) entries.    static mapping(string|int:mixed) cache = ([ ]);    -  +  //! Cache of cache-ids that have no expiration time. +  //! This cache is maintained in sync with @[cache]. +  //! Note that entries not in this cache may still have +  //! unlimited expiration time. +  static mapping(string|int:int) no_expiry = ([ ]); +     static void setup_table()    {    // New style argument2 table.    if(catch(QUERY("SELECT id FROM "+name+"2 LIMIT 0")))    {    master()->resolv("DBManager.is_module_table")    ( 0, "local", name+"2",    "The argument cache, used to map between "    "a unique string and an argument mapping" );    catch(QUERY("DROP TABLE "+name+"2" ));    QUERY("CREATE TABLE "+name+"2 ("    "id CHAR(32) PRIMARY KEY, "    "ctime DATETIME NOT NULL, "    "atime DATETIME NOT NULL, " -  "contents MEDIUMBLOB NOT NULL)"); +  "timeout INT NULL, " +  "contents MEDIUMBLOB NOT NULL, " +  " INDEX(timeout))");    }    -  +  if (catch (QUERY ("SELECT timeout FROM " + name + "2 LIMIT 0"))) +  { +  // Upgrade a table without timeout. +  QUERY ("ALTER TABLE " + name + "2 " +  " ADD timeout INT NULL " +  "AFTER atime"); +  QUERY ("ALTER TABLE " + name + "2 " +  " ADD INDEX(timeout)"); +  } +     catch {    array(mapping(string:mixed)) res =    QUERY("DESCRIBE "+name+"2 contents");       if(res[0]->Type == "blob") {    QUERY("ALTER TABLE "+name+"2 MODIFY contents MEDIUMBLOB NOT NULL");    werror("ArgCache: Extending \"contents\" field in table \"%s2\" from BLOB to MEDIUMBLOB.\n", name);    }    };    }    -  +  static void do_cleanup() +  { +  QUERY("DELETE FROM " + name + "2 " +  " WHERE timeout IS NOT NULL " +  " AND timeout < %d", time()); +  } +  +  static void cleanup_process( ) +  { +  // Flushes may be costly in large sites (since there's no index +  // on the timeout field) so schedule next run sometime after +  // 04:30 the day after tomorrow. +  int now = time(); +  mapping info = localtime(now); +  int wait = (int) ((24 - info->hour) + 24 + 4.5) * 3600 + random(500); +  background_run(wait, cleanup_process); +  +  do_cleanup(); +  } +     static void init_db()    {    // Delay DBManager resolving to before the 'roxen' object is    // compiled.    cache = ([]); -  +  no_expiry = ([]);    db = dbm_cached_get("local");    setup_table( ); -  +  +  // Cleanup exprired entries on start. +  background_run( 10, cleanup_process );    }       static void create( string _name )    {    name = _name;    init_db();    // Support that the 'local' database moves (not really nessesary,    // but it won't hurt either)    master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db );    get_plugins();
Roxen.git/server/base_server/roxen.pike:3967:    " WHERE id = %s", id);    if(!sizeof(res))    return 0;    if (!dont_update_atime)    QUERY("UPDATE "+name+"2 "    " SET atime = NOW() "    " WHERE id = %s", id);    return res[0]->contents;    }    -  void create_key( string id, string encoded_args ) +  void create_key( string id, string encoded_args, int|void timeout )    { -  +  if (!zero_type(timeout) && (timeout < time(1))) return; // Expired.    LOCK();    array(mapping) rows =    QUERY("SELECT id, contents FROM "+name+"2 WHERE id = %s", id );    foreach( rows, mapping row )    if( row->contents != encoded_args ) {    report_error("ArgCache.create_key(): "    "Duplicate key found! Please report this to support@roxen.com: "    "id: %O, old data: %O, new data: %O\n",    id, row->contents, encoded_args);    error("ArgCache.create_key() Duplicate key found!\n");    }    -  if(sizeof(rows)) +  if(sizeof(rows)) { +  if (zero_type(timeout)) { +  QUERY("UPDATE "+name+"2 " +  " SET timeout = NULL " +  " WHERE id = %s", id); +  } else { +  QUERY("UPDATE "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s " +  " AND timeout IS NOT NULL " +  " AND timeout < %d", +  timeout, id, timeout); +  }    return; -  +  }       QUERY( "INSERT INTO "+name+"2 "    "(id, contents, ctime, atime) VALUES "    "(%s, " MYSQL__BINARY "%s, NOW(), NOW())", id, encoded_args );    -  +  if (!zero_type(timeout)) { +  QUERY("UPDATE "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s", +  timeout, id); +  } +     dwerror("ArgCache: Create new key %O\n", id);       (plugins->create_key-({0}))( id, encoded_args );    }       static array plugins;    static void get_plugins()    {    plugins = ({});    foreach( ({ "../local/arg_cache_plugins", "arg_cache_plugins" }), string d)
Roxen.git/server/base_server/roxen.pike:4016:       mapping plugins_read_encoded_args( string id )    {    mapping args;    foreach( (plugins->read_encoded_args - ({0})), function(string:mapping) f )    if( args = f( id ) )    return args;    return 0;    }    -  string store( mapping args ) +  string store( mapping args, int|void timeout )    //! Store a mapping (of purely encode_value:able data) in the    //! argument cache. The string returned is your key to retrieve the    //! data later. -  +  //! +  //! @param timeout +  //! Timeout for the entry in seconds from now. If @expr{UNDEFINED@}, +  //! the entry will not expire.    { -  +  if (!zero_type(timeout)) timeout += time(1);    string encoded_args = encode_value_canonic( args );    string id = Gmp.mpz(Crypto.sha()->update(encoded_args)->digest(), 256)->digits(36); -  if( cache[ id ] ) +  if( cache[ id ] ) { +  if (!no_expiry[id]) { +  // The cache id may have a timeout. +  if (zero_type(timeout)) { +  // No timeout now, but there may have been one earlier. +  QUERY("UPDATE "+name+"2 " +  " SET timeout = NULL " +  " WHERE id = %s " +  " AND timeout IS NOT NULL", id); +  no_expiry[id] = 1; +  } else { +  // Attempt to bump the timeout. +  QUERY("UPDATE "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s " +  " AND timeout IS NOT NULL " +  " AND timeout < %d", +  timeout, id, timeout); +  } +  }    return id; -  create_key(id, encoded_args); +  } +  create_key(id, encoded_args, timeout); +  if( sizeof( cache ) >= CACHE_SIZE ) { +  cache = ([]); +  no_expiry = ([]); +  }    if( !cache[ id ] )    cache[ id ] = args+([]); -  if( sizeof( cache ) >= CACHE_SIZE ) -  cache = ([]); +  if (zero_type(timeout)) { +  no_expiry[ id ] = 1; +  }    return id;    }          mapping lookup( string id )    //! Recall a mapping stored in the cache.    {    if( cache[id] )    return cache[id] + ([]);    string encoded_args = read_encoded_args(id) || plugins_read_encoded_args(id);    if(!encoded_args) {    error("Requesting unknown key (not found in db)\n");    }    mapping args = decode_value(encoded_args); -  cache[id] = args + ([]); -  if( sizeof( cache ) >= CACHE_SIZE ) +  if( sizeof( cache ) >= CACHE_SIZE ) {    cache = ([]); -  +  no_expiry = ([]); +  } +  cache[id] = args + ([]);    return args;    }       void delete( string id )    //! Remove the data element stored under the key 'id'.    {    LOCK();    (plugins->delete-({0}))( id );    m_delete( cache, id );   
Roxen.git/server/base_server/roxen.pike:4077:    constant FETCH_ROWS = 10000;       // The server does only need to use file based argcache    // replication if the server don't participate in a replicate    // setup with a shared database.    if( !has_value((plugins->is_functional-({0}))(), 1) )    {    int cursor;    array(string) ids;    do { +  array(mapping(string:string)) entries;    if(from_time)    // Only replicate entries accessed during the prefetch crawling. -  ids = +  entries =    (array(string)) -  QUERY( "SELECT id from "+name+"2 " +  QUERY( "SELECT id, timeout from "+name+"2 "    " WHERE atime >= FROM_UNIXTIME(%d) " -  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS)->id; +  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS);    else    // Make sure _every_ entry is replicated when a dump is created. -  ids = +  entries =    (array(string)) -  QUERY( "SELECT id from "+name+"2 " -  " LIMIT %d, %d", cursor, FETCH_ROWS)->id; +  QUERY( "SELECT id, timeout from "+name+"2 " +  " LIMIT %d, %d", cursor, FETCH_ROWS);    -  +  ids = entries->id; +  array(string) timeouts = entries->timeout;    cursor += FETCH_ROWS;    -  foreach(ids, string id) { +  foreach(ids; int i; string id) {    dwerror("ArgCache.write_dump(): %O\n", id);    -  string s = -  MIME.encode_base64(encode_value(({ id, read_encoded_args(id, 1) })), -  1)+"\n"; +  string s; +  if (timeouts[i]) { +  int timeout = (int)timeouts[i]; +  if (timeout < time(1)) { +  // Expired entry. Don't replicate. +  continue; +  } +  s = +  MIME.encode_base64(encode_value(({ id, read_encoded_args(id, 1), +  timeout })), 1)+"\n"; +  } else { +  // No timeout. Backward-compatible format. +  s = +  MIME.encode_base64(encode_value(({ id, read_encoded_args(id, 1) +  })), 1)+"\n"; +  }    if(sizeof(s) != file->write(s))    return 0;    }    } while(sizeof(ids) == FETCH_ROWS);    }    return file->write("EOF\n") == 4;    }       string read_dump (Stdio.FILE file)    // Returns an error message if there was a parse error, 0 otherwise.
Roxen.git/server/base_server/roxen.pike:4141:    // index_id when from_time was zero, so we ignore them here    // instead.    dwerror ("ArgCache.read_dump(): entry ignored.\n");    else {    array v = decode_value(a[1]), i = decode_value(a[3]);   #if 0    dwerror ("ArgCache.read_dump(): values: %O, indices: %O\n", v, i);   #endif    store(mkmapping(i, v));    } -  } else if (sizeof(a) == 2) { -  // New style argcache dump. +  } else if ((sizeof(a) == 2) || (sizeof(a) == 3)) { +  // New style argcache dump, possibly with timeout.    dwerror("ArgCache.read_dump(): %O\n", a[0]); -  create_key(a[0], a[1]); +  create_key(@a);    } else    return "Decode failed for argcache record (wrong size on key array)\n";    }    if(s != "EOF")    return "Missing data in argcache file\n";    return 0;    }       void refresh_arg(string id)    {