Branch: Tag:

2009-11-26

2009-11-26 15:43:48 by Henrik Grubbström (Grubba) <grubba@grubba.org>

Backported support for ArgCache timeouts from Roxen 5.0. Fixes [bug 4548 (#4548)].

Rev: server/base_server/roxen.pike:1.966
Rev: server/modules/graphics/atlas.pike:1.18
Rev: server/modules/graphics/business.pike:1.152
Rev: server/modules/graphics/cimg.pike:1.80
Rev: server/modules/graphics/gbutton.pike:1.116
Rev: server/modules/graphics/graphic_text.pike:1.305
Rev: server/modules/graphics/gxml.pike:1.39

6:   // Per Hedbor, Henrik Grubbström, Pontus Hagland, David Hedbor and others.   // ABS and suicide systems contributed freely by Francesco Chemolli    - constant cvs_version="$Id: roxen.pike,v 1.965 2009/06/29 13:22:54 mast Exp $"; + constant cvs_version="$Id: roxen.pike,v 1.966 2009/11/26 15:36:52 grubba Exp $";      //! @appears roxen   //!
2880:    mapping meta;    string data;    array guides; + #ifdef ARG_CACHE_DEBUG +  werror("draw args: %O\n", args ); + #endif    mixed reply = draw_function( @copy_value(args), id );       if( !reply ) {
3629:       mapping http_file_answer( array|string|mapping data,    RequestID id, -  int|void nodraw ) +  int|void nodraw, int|void timeout )    //! Returns a @[result mapping] like one generated by    //! @[Roxen.http_file_answer()] but for the image file    //! rendered from the `data' instructions.
3640:    //! not found' error)    {    current_configuration->set(id->conf); -  string na = store( data,id ); +  string na = store( data, id, timeout );    mixed res;   #ifdef ARG_CACHE_DEBUG -  werror("data %O\n", na ); +  werror("data: %O id: %O\n", na, id );   #endif    if(! (res=restore( na,id )) )    {
3729:    return ([ "":what ]);    }    -  string store( array|string|mapping data, RequestID id ) +  string store( array|string|mapping data, RequestID id, int|void timeout )    //! Store the data your draw callback expects to receive as its    //! first argument(s). If the data is an array, the draw callback    //! will be called like <pi>callback( @@data, id )</pi>. -  +  //! +  //! @param timeout +  //! Timeout for the entry in seconds from now. If @expr{UNDEFINED@}, +  //! the entry will not expire. Currently just passed along to +  //! the @[ArgCache].    {    string ci, user;    function update_args = lambda ( mapping a )
3754:    if( mappingp( data ) )    {    update_args( data ); -  ci = argcache->store( data ); +  ci = argcache->store( data, timeout );    }    else if( arrayp( data ) )    {    if( !mappingp( data[0] ) )    error("Expected mapping as the first element of the argument array\n");    update_args( data[0] ); -  ci = map( map( data, tomapp ), argcache->store )*"$"; +  ci = map( map( data, tomapp ), argcache->store, timeout )*"$";    } else    ci = data;    update_args = 0; // To avoid garbage.
3911:   #define dwerror(ARGS...) 0   #endif    +  //! Cache of the latest entries requested or stored. +  //! Limited to @[CACHE_SIZE] (currently @expr{900@}) entries.    static mapping(string|int:mixed) cache = ([ ]);    -  +  //! Cache of cache-ids that have no expiration time. +  //! This cache is maintained in sync with @[cache]. +  //! Note that entries not in this cache may still have +  //! unlimited expiration time. +  static mapping(string|int:int) no_expiry = ([ ]); +     static void setup_table()    {    // New style argument2 table.
3927:    "id CHAR(32) PRIMARY KEY, "    "ctime DATETIME NOT NULL, "    "atime DATETIME NOT NULL, " -  "contents MEDIUMBLOB NOT NULL)"); +  "timeout INT NULL, " +  "contents MEDIUMBLOB NOT NULL, " +  " INDEX(timeout))");    }    -  +  if (catch (QUERY ("SELECT timeout FROM " + name + "2 LIMIT 0"))) +  { +  // Upgrade a table without timeout. +  QUERY ("ALTER TABLE " + name + "2 " +  " ADD timeout INT NULL " +  "AFTER atime"); +  QUERY ("ALTER TABLE " + name + "2 " +  " ADD INDEX(timeout)"); +  } +     catch {    array(mapping(string:mixed)) res =    QUERY("DESCRIBE "+name+"2 contents");
3941:    };    }    +  static void do_cleanup() +  { +  QUERY("DELETE FROM " + name + "2 " +  " WHERE timeout IS NOT NULL " +  " AND timeout < %d", time()); +  } +  +  static void cleanup_process( ) +  { +  // Flushes may be costly in large sites (since there's no index +  // on the timeout field) so schedule next run sometime after +  // 04:30 the day after tomorrow. +  int now = time(); +  mapping info = localtime(now); +  int wait = (int) ((24 - info->hour) + 24 + 4.5) * 3600 + random(500); +  background_run(wait, cleanup_process); +  +  do_cleanup(); +  } +     static void init_db()    {    // Delay DBManager resolving to before the 'roxen' object is    // compiled.    cache = ([]); -  +  no_expiry = ([]);    db = dbm_cached_get("local");    setup_table( ); -  +  +  // Cleanup exprired entries on start. +  background_run( 10, cleanup_process );    }       static void create( string _name )
3974:    return res[0]->contents;    }    -  void create_key( string id, string encoded_args ) +  void create_key( string id, string encoded_args, int|void timeout )    { -  +  if (!zero_type(timeout) && (timeout < time(1))) return; // Expired.    LOCK();    array(mapping) rows =    QUERY("SELECT id, contents FROM "+name+"2 WHERE id = %s", id );
3988:    error("ArgCache.create_key() Duplicate key found!\n");    }    -  if(sizeof(rows)) +  if(sizeof(rows)) { +  if (zero_type(timeout)) { +  QUERY("UPDATE "+name+"2 " +  " SET timeout = NULL " +  " WHERE id = %s", id); +  } else { +  QUERY("UPDATE "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s " +  " AND timeout IS NOT NULL " +  " AND timeout < %d", +  timeout, id, timeout); +  }    return; -  +  }       QUERY( "INSERT INTO "+name+"2 "    "(id, contents, ctime, atime) VALUES "    "(%s, " MYSQL__BINARY "%s, NOW(), NOW())", id, encoded_args );    -  +  if (!zero_type(timeout)) { +  QUERY("UPDATE "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s", +  timeout, id); +  } +     dwerror("ArgCache: Create new key %O\n", id);       (plugins->create_key-({0}))( id, encoded_args );
4023:    return 0;    }    -  string store( mapping args ) +  string store( mapping args, int|void timeout )    //! Store a mapping (of purely encode_value:able data) in the    //! argument cache. The string returned is your key to retrieve the    //! data later. -  +  //! +  //! @param timeout +  //! Timeout for the entry in seconds from now. If @expr{UNDEFINED@}, +  //! the entry will not expire.    { -  +  if (!zero_type(timeout)) timeout += time(1);    string encoded_args = encode_value_canonic( args );    string id = Gmp.mpz(Crypto.sha()->update(encoded_args)->digest(), 256)->digits(36); -  if( cache[ id ] ) +  if( cache[ id ] ) { +  if (!no_expiry[id]) { +  // The cache id may have a timeout. +  if (zero_type(timeout)) { +  // No timeout now, but there may have been one earlier. +  QUERY("UPDATE "+name+"2 " +  " SET timeout = NULL " +  " WHERE id = %s " +  " AND timeout IS NOT NULL", id); +  no_expiry[id] = 1; +  } else { +  // Attempt to bump the timeout. +  QUERY("UPDATE "+name+"2 " +  " SET timeout = %d " +  " WHERE id = %s " +  " AND timeout IS NOT NULL " +  " AND timeout < %d", +  timeout, id, timeout); +  } +  }    return id; -  create_key(id, encoded_args); +  } +  create_key(id, encoded_args, timeout); +  if( sizeof( cache ) >= CACHE_SIZE ) { +  cache = ([]); +  no_expiry = ([]); +  }    if( !cache[ id ] )    cache[ id ] = args+([]); -  if( sizeof( cache ) >= CACHE_SIZE ) -  cache = ([]); +  if (zero_type(timeout)) { +  no_expiry[ id ] = 1; +  }    return id;    }   
4051:    error("Requesting unknown key (not found in db)\n");    }    mapping args = decode_value(encoded_args); -  cache[id] = args + ([]); -  if( sizeof( cache ) >= CACHE_SIZE ) +  if( sizeof( cache ) >= CACHE_SIZE ) {    cache = ([]); -  +  no_expiry = ([]); +  } +  cache[id] = args + ([]);    return args;    }   
4084:    int cursor;    array(string) ids;    do { +  array(mapping(string:string)) entries;    if(from_time)    // Only replicate entries accessed during the prefetch crawling. -  ids = +  entries =    (array(string)) -  QUERY( "SELECT id from "+name+"2 " +  QUERY( "SELECT id, timeout from "+name+"2 "    " WHERE atime >= FROM_UNIXTIME(%d) " -  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS)->id; +  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS);    else    // Make sure _every_ entry is replicated when a dump is created. -  ids = +  entries =    (array(string)) -  QUERY( "SELECT id from "+name+"2 " -  " LIMIT %d, %d", cursor, FETCH_ROWS)->id; +  QUERY( "SELECT id, timeout from "+name+"2 " +  " LIMIT %d, %d", cursor, FETCH_ROWS);    -  +  ids = entries->id; +  array(string) timeouts = entries->timeout;    cursor += FETCH_ROWS;    -  foreach(ids, string id) { +  foreach(ids; int i; string id) {    dwerror("ArgCache.write_dump(): %O\n", id);    -  string s = -  MIME.encode_base64(encode_value(({ id, read_encoded_args(id, 1) })), -  1)+"\n"; +  string s; +  if (timeouts[i]) { +  int timeout = (int)timeouts[i]; +  if (timeout < time(1)) { +  // Expired entry. Don't replicate. +  continue; +  } +  s = +  MIME.encode_base64(encode_value(({ id, read_encoded_args(id, 1), +  timeout })), 1)+"\n"; +  } else { +  // No timeout. Backward-compatible format. +  s = +  MIME.encode_base64(encode_value(({ id, read_encoded_args(id, 1) +  })), 1)+"\n"; +  }    if(sizeof(s) != file->write(s))    return 0;    }
4148:   #endif    store(mkmapping(i, v));    } -  } else if (sizeof(a) == 2) { -  // New style argcache dump. +  } else if ((sizeof(a) == 2) || (sizeof(a) == 3)) { +  // New style argcache dump, possibly with timeout.    dwerror("ArgCache.read_dump(): %O\n", a[0]); -  create_key(a[0], a[1]); +  create_key(@a);    } else    return "Decode failed for argcache record (wrong size on key array)\n";    }