pike.git
/
lib
/
modules
/
Search.pmod
/
Queue.pmod
/
MySQL.pike
version
»
Context lines:
10
20
40
80
file
none
3
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:15:
// performing sql queries elsewhere. Sql.Sql ret = _db->get(); if (ret && !ret->ping()) return ret; return _db->set(Sql.Sql( url )); } Web.Crawler.Stats stats; Web.Crawler.Policy policy; Web.Crawler.RuleSet allow, deny;
-
static
string to_md5(string url)
+
protected
string to_md5(string url)
{
-
Crypto.MD5
md5 =
Crypto.MD5(
);
-
md5->update(
string_to_utf8(url))
;
-
return String.string2hex(md5->digest(
)
)
;
+
return String.string2hex(
Crypto.MD5.
hash
(string_to_utf8(url)));
} //! @param _url //! @[Sql.Sql] URL for the database to store the queue. //! //! @param _table //! @[Sql.Sql] table name to store the queue in. //! //! If the table doesn't exist it will be created. void create( Web.Crawler.Stats _stats,
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:45:
void|Web.Crawler.RuleSet _deny) { stats = _stats; policy = _policy; allow=_allow; deny=_deny; table = _table; url = _url; perhaps_create_table( ); }
-
static
void perhaps_create_table( )
+
protected
void perhaps_create_table( )
{ db->query( #" create table IF NOT EXISTS "+table+#" ( uri blob not null, uri_md5 char(32) not null default '', template varchar(255) not null default '', md5 char(32) not null default '', recurse tinyint not null, stage tinyint not null,
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:67:
INDEX stage (stage), INDEX uri (uri(255)) ) "); if (!((multiset)db->query("SHOW INDEX FROM " + table)->Key_name)["uri"]) { db->query("ALTER TABLE " + table + " ADD INDEX uri (uri(255))"); } }
-
static
mapping hascache = ([]);
+
protected
mapping hascache = ([]);
void clear_cache() { hascache = ([]); }
-
static
int has_uri( string|Standards.URI uri )
+
protected
int has_uri( string|Standards.URI uri )
{ uri = (string)uri; if( sizeof(hascache) > 100000 ) hascache = ([]); return hascache[uri]|| (hascache[uri]= sizeof(db->query("select stage from "+table+" where uri_md5=%s", to_md5(uri)))); } void add_uri( Standards.URI uri, int recurse, string template, void|int force )
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:155:
mapping(string:mapping(string:string)) extra_data = ([]); mapping get_extra( Standards.URI uri ) { if( extra_data[(string)uri] ) return extra_data[(string)uri] || ([ ]); array r = db->query( "SELECT md5,recurse,stage,template " "FROM "+table+" WHERE uri_md5=%s", to_md5((string)uri) ); return (sizeof(r) && r[0]) || ([ ]); }
-
static
int empty_count;
-
static
int retry_count;
+
protected
int empty_count;
+
protected
int retry_count;
// cache, for performance reasons.
-
static
array possible=({});
-
static
int p_c;
+
protected
array possible=({});
+
protected
int p_c;
int|Standards.URI get() { if(stats->concurrent_fetchers() > policy->max_concurrent_fetchers) return -1; if( sizeof( possible ) <= p_c ) { p_c = 0; possible = db->query( "select * from "+table+" where stage=0 limit 20" );