pike.git / lib / modules / Search.pmod / Queue.pmod / MySQL.pike

version» Context lines:

pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:5:      Web.Crawler.Stats stats;   Web.Crawler.Policy policy;   Web.Crawler.RuleSet allow, deny;      inherit Web.Crawler.Queue;      static string to_md5(string url)   {    object md5 = Crypto.md5(); -  md5->update(url); +  md5->update(string_to_utf8(url));    return Crypto.string_to_hex(md5->digest());   }      void create( Web.Crawler.Stats _stats,    Web.Crawler.Policy _policy,       string _url, string _table,       void|Web.Crawler.RuleSet _allow,    void|Web.Crawler.RuleSet _deny)
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:76:    // FIXME: Make these configurable?    foreach( ({"index.xml", "index.html", "index.htm"}),    string index)    if(search(rpath,reverse(index))==0)    rpath=rpath[sizeof(index)..];    r->path=reverse(rpath);       if( force || (check_link(uri, allow, deny) && !has_uri( r ) ))    db->query( "insert into "+table+    " (uri,uri_md5,recurse,template) values (%s,%s,%d,%s)", -  (string)r, to_md5((string)r), recurse, (template||"") ); +  string_to_utf8((string)r), +  to_md5((string)r), recurse, (template||"") );   }      void set_md5( Standards.URI uri, string md5 )   {    db->query( "update "+table+    " set md5=%s WHERE uri_md5=%s", md5, to_md5((string)uri) );   }      mapping(string:mapping(string:string)) extra_data = ([]);   mapping get_extra( Standards.URI uri )
pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:115:   {    if(stats->concurrent_fetchers() > policy->max_concurrent_fetchers)    {    return -1;    }       if( sizeof( possible ) <= p_c )    {    p_c = 0;    possible = db->query( "select * from "+table+" where stage=0 limit 20" ); -  extra_data = mkmapping( possible->uri, possible ); -  possible = possible->uri; +  extra_data = mkmapping( map(possible->uri,utf8_to_string), possible ); +  possible = map(possible->uri,utf8_to_string);    }       while( sizeof( possible ) > p_c )    {    empty_count=0;    if( possible[ p_c ] )    {    Standards.URI ur = Standards.URI( possible[p_c++] );       if( stats->concurrent_fetchers( ur->host ) >