pike.git / lib / modules / Search.pmod / Queue.pmod / MySQL.pike

version» Context lines:

pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:15:    // performing sql queries elsewhere.    Sql.Sql ret = _db->get();    if (ret && !ret->ping()) return ret;    return _db->set(Sql.Sql( url ));   }      Web.Crawler.Stats stats;   Web.Crawler.Policy policy;   Web.Crawler.RuleSet allow, deny;    - static string to_md5(string url) + protected string to_md5(string url)   { -  Crypto.MD5 md5 = Crypto.MD5(); -  md5->update(string_to_utf8(url)); -  return String.string2hex(md5->digest()); +  return String.string2hex(Crypto.MD5.hash(string_to_utf8(url)));   }      //! @param _url   //! @[Sql.Sql] URL for the database to store the queue.   //!   //! @param _table   //! @[Sql.Sql] table name to store the queue in.   //!   //! If the table doesn't exist it will be created.   void create( Web.Crawler.Stats _stats,
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:45:    void|Web.Crawler.RuleSet _deny)   {    stats = _stats; policy = _policy;    allow=_allow; deny=_deny;    table = _table;    url = _url;       perhaps_create_table( );   }    - static void perhaps_create_table( ) + protected void perhaps_create_table( )   {    db->query(   #"    create table IF NOT EXISTS "+table+#" (    uri blob not null,    uri_md5 char(32) not null default '',    template varchar(255) not null default '',    md5 char(32) not null default '',    recurse tinyint not null,    stage tinyint not null,
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:67:    INDEX stage (stage),    INDEX uri (uri(255))    )    ");    if (!((multiset)db->query("SHOW INDEX FROM " + table)->Key_name)["uri"]) {    db->query("ALTER TABLE " + table +    " ADD INDEX uri (uri(255))");    }   }    - static mapping hascache = ([]); + protected mapping hascache = ([]);      void clear_cache()   {    hascache = ([]);   }    - static int has_uri( string|Standards.URI uri ) + protected int has_uri( string|Standards.URI uri )   {    uri = (string)uri;    if( sizeof(hascache) > 100000 ) hascache = ([]);    return hascache[uri]||    (hascache[uri]=    sizeof(db->query("select stage from "+table+" where uri_md5=%s",    to_md5(uri))));   }      void add_uri( Standards.URI uri, int recurse, string template, void|int force )
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:155:   mapping(string:mapping(string:string)) extra_data = ([]);   mapping get_extra( Standards.URI uri )   {    if( extra_data[(string)uri] )    return extra_data[(string)uri] || ([ ]);    array r = db->query( "SELECT md5,recurse,stage,template "    "FROM "+table+" WHERE uri_md5=%s", to_md5((string)uri) );    return (sizeof(r) && r[0]) || ([ ]);   }    - static int empty_count; - static int retry_count; + protected int empty_count; + protected int retry_count;      // cache, for performance reasons. - static array possible=({}); - static int p_c; + protected array possible=({}); + protected int p_c;      int|Standards.URI get()   {    if(stats->concurrent_fetchers() > policy->max_concurrent_fetchers)    return -1;       if( sizeof( possible ) <= p_c )    {    p_c = 0;    possible = db->query( "select * from "+table+" where stage=0 limit 20" );