pike.git / lib / modules / Search.pmod / Queue.pmod / MySQL.pike

version» Context lines:

pike.git/lib/modules/Search.pmod/Queue.pmod/MySQL.pike:119:   static int empty_count;   static int retry_count;      // cache, for performance reasons.   static array possible=({});   static int p_c;      int|Standards.URI get()   {    if(stats->concurrent_fetchers() > policy->max_concurrent_fetchers) -  { +     return -1; -  } +        if( sizeof( possible ) <= p_c )    {    p_c = 0;    possible = db->query( "select * from "+table+" where stage=0 limit 20" );    extra_data = mkmapping( map(possible->uri,utf8_to_string), possible );    possible = map(possible->uri,utf8_to_string);    }       while( sizeof( possible ) > p_c )    {    empty_count=0;    if( possible[ p_c ] )    { -  Standards.URI ur = Standards.URI( possible[p_c++] ); +  Standards.URI uri = Standards.URI( possible[p_c++] );    -  if( stats->concurrent_fetchers( ur->host ) > +  if( stats->concurrent_fetchers( uri->host ) >    policy->max_concurrent_fetchers_per_host )    {    retry_count++;    continue; // not this host..    }    possible[p_c-1] = 0;    retry_count=0; -  set_stage( ur, 1 ); -  return ur; +  set_stage( uri, 1 ); +  return uri;    }    p_c++;    continue;    }       if( stats->concurrent_fetchers() )    {    return -1;    } -  +     // delay for (quite) a while. -  if( empty_count++ > 40 ) -  { -  if( num_with_stage( 2 ) || num_with_stage( 3 ) ) -  { -  empty_count=0; -  return -1; -  } + // if( empty_count++ > 40 ) + // { + // if( num_with_stage( 2 ) || num_with_stage( 3 ) ) + // { + // empty_count=0; + // werror("Queue: delay for (quite) a while.\n"); + // werror("possible: %O\np_c: %O\n", possible, p_c); + // return -1; + // } + // return 0; + // }    return 0;   }    -  return -1; - } -  +    void put(string|array(string)|Standards.URI|array(Standards.URI) uri)   {    if(arrayp(uri))    {    foreach(uri, string|object _uri)    put(_uri);    return;    }    if(!objectp(uri))    uri=Standards.URI(uri);       add_uri( uri, 1, 0 );   }    - void done( Standards.URI uri, -  int called ) - { -  if( called ) -  set_stage( uri, 2 ); -  else -  set_stage( uri, 5 ); - } -  +    void clear()   {    hascache = ([ ]);    db->query("delete from "+table);   }      void remove_uri(string|Standards.URI uri)   {    db->query("delete from "+table+" where uri_md5=%s", to_md5((string)uri));   }