Roxen.git / server / base_server / roxen.pike

version» Context lines:

Roxen.git/server/base_server/roxen.pike:1:   // This file is part of Roxen WebServer.   // Copyright © 1996 - 2009, Roxen IS.   //   // The Roxen WebServer main program.   //   // Per Hedbor, Henrik Grubbström, Pontus Hagland, David Hedbor and others.   // ABS and suicide systems contributed freely by Francesco Chemolli    - constant cvs_version="$Id: roxen.pike,v 1.1080 2011/02/15 13:51:39 marty Exp $"; + constant cvs_version="$Id$";      //! @appears roxen   //!   //! The Roxen WebServer main program.      // The argument cache. Used by the image cache.   ArgCache argcache;      // Some headerfiles   #define IN_ROXEN
Roxen.git/server/base_server/roxen.pike:26:   // Inherits   inherit "global_variables";   #ifdef SNMP_AGENT   inherit "snmpagent";   #endif   #ifdef SMTP_RELAY   inherit "smtprelay";   #endif   inherit "hosts";   inherit "disk_cache"; + inherit "fsgc";   // inherit "language";   inherit "supports";   inherit "module_support";   inherit "config_userdb";      // Used to find out which thread is the backend thread.   Thread.Thread backend_thread;      // --- Locale defines ---   
Roxen.git/server/base_server/roxen.pike:107:       else if( !fname ) {    fname = master()->program_name( o );    if (!fname)    return ({0, 0});    }       string cwd = getcwd() + "/";    if (has_prefix (fname, cwd))    fname = fname[sizeof (cwd)..]; +  else if (has_prefix (fname, roxenloader.server_dir)) +  fname = fname[sizeof (roxenloader.server_dir)..];       return ({fname, line});   }      string filename( program|object o )   {    [string fname, int line] = filename_2 (o);    return fname || "(unknown program)";   }   
Roxen.git/server/base_server/roxen.pike:130:   // cache static optimization for tags such as <if> and <emit> inside   // <cache> since that optimization can give tricky incompatibilities   // with 2.4.   array(string) compat_levels = ({"2.1", "2.2", "2.4", "2.5",    "3.3", "3.4",    "4.0", "4.5",    "5.0", "5.1", "5.2" });      #ifdef THREADS   mapping(string:string) thread_names = ([]); - string thread_name( object thread ) + string thread_name( object thread, int|void skip_auto_name )   {    string tn; -  if( thread_names[ tn=sprintf("%O",thread) ] ) +  if( thread_names[ tn=sprintf("%O",thread) ] || skip_auto_name )    return thread_names[tn];    return tn;   }      void name_thread( object thread, string name )   { -  thread_names[ sprintf( "%O", thread ) ] = name; +  string th_key = sprintf("%O", thread); +  if (name) +  thread_names[th_key] = name; +  else +  m_delete(thread_names, th_key);   }      // This mutex is used by Privs   Thread.Mutex euid_egid_lock = Thread.Mutex();   #endif /* THREADS */      /*    * The privilege changer. Works like a mutex lock, but changes the UID/GID    * while held. Blocks all threads.    *
Roxen.git/server/base_server/roxen.pike:208: Inside #if defined(PRIVS_DEBUG)
   "privs_level: %O\n",    reason, uid, gid, privs_level));   #endif /* PRIVS_DEBUG */      #ifdef HAVE_EFFECTIVE_USER    array u;      #ifdef THREADS    if (euid_egid_lock) {    if (mixed err = catch { mutex_key = euid_egid_lock->lock(); }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);    }    threads_disabled = _disable_threads();   #endif /* THREADS */       p_level = privs_level++;       /* Needs to be here since root-priviliges may be needed to    * use getpw{uid,nam}.    */    saved_uid = geteuid();
Roxen.git/server/base_server/roxen.pike:269: Inside #if defined(HAVE_EFFECTIVE_USER)
      if(LOGP)    report_notice(LOC_M(1, "Change to %s(%d):%d privs wanted (%s), from %s"),    (string)u[0], (int)uid, (int)gid,    (string)reason,    (string)dbt(backtrace()[-2]));       if (u[2]) {   #if efun(cleargroups)    if (mixed err = catch { cleargroups(); }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);   #endif /* cleargroups */   #if efun(initgroups)    if (mixed err = catch { initgroups(u[0], u[3]); }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);   #endif    }    gid = gid || getgid();    int err = (int)setegid(new_gid = gid);    if (err < 0) {    report_warning(LOC_M(2, "Privs: WARNING: Failed to set the "    "effective group id to %d!\n"    "Check that your password database is correct "    "for user %s(%d),\n and that your group "    "database is correct.\n"),
Roxen.git/server/base_server/roxen.pike:366: Inside #if defined(HAVE_EFFECTIVE_USER)
   if (mixed err = catch {    array bt = backtrace();    if (sizeof(bt) >= 2) {    report_notice(LOC_M(3,"Change back to uid#%d gid#%d, from %s")+"\n",    saved_uid, saved_gid, dbt(bt[-2]));    } else {    report_notice(LOC_M(4,"Change back to uid#%d gid#%d, "    "from backend")+"\n", saved_uid, saved_gid);    }    }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);    }      #ifdef PRIVS_DEBUG    int uid = geteuid();    if (uid != new_uid) {    report_debug("Privs: UID #%d differs from expected #%d\n"    "%s\n",    uid, new_uid, describe_backtrace(backtrace()));    }    int gid = getegid();
Roxen.git/server/base_server/roxen.pike:388: Inside #if defined(HAVE_EFFECTIVE_USER) and #if defined(PRIVS_DEBUG)
   report_debug("Privs: GID #%d differs from expected #%d\n"    "%s\n",    gid, new_gid, describe_backtrace(backtrace()));    }   #endif /* PRIVS_DEBUG */       seteuid(0);    array u = getpwuid(saved_uid);   #if efun(cleargroups)    if (mixed err = catch { cleargroups(); }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);   #endif /* cleargroups */    if(u && (sizeof(u) > 3)) {    if (mixed err = catch { initgroups(u[0], u[3]); }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);    }    setegid(saved_gid);    seteuid(saved_uid);    enable_coredumps(1);   #endif /* HAVE_EFFECTIVE_USER */    }   #else /* efun(seteuid) */    void create(string reason, int|string|void uid, int|string|void gid){}   #endif /* efun(seteuid) */   }
Roxen.git/server/base_server/roxen.pike:470:    werror (describe_backtrace (err));   #endif /* THREADS */    if (!exit_code || once_mode) {    // We're shutting down; Attempt to take mysqld with us.    if (mixed err =    catch { report_notice("Shutting down MySQL.\n"); } ||    catch {    Sql.Sql db = connect_to_my_mysql(0, "mysql");    db->shutdown();    }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);    }    // Zap some of the remaining caches.    destruct (argcache);    destruct (cache);   #if 0    // Disabled since it's lying when the server is shut down with a    // SIGTERM or SIGINT to the start script (which include the stop    // action of the init.d script).    if (mixed err = catch {    if (exit_code && !once_mode)    report_notice("Restarting Roxen.\n");    else    report_notice("Shutting down Roxen.\n");    }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);   #endif    roxenloader.real_exit( exit_code ); // Now we die...   }      private int shutdown_recurse;      // Shutdown Roxen   // exit_code = 0 True shutdown   // exit_code = -1 Restart   private void low_shutdown(int exit_code)   {    if(shutdown_recurse >= 4)    {    if (mixed err =    catch (report_notice("Exiting roxen (spurious signals received).\n")) ||    catch (stop_all_configurations())) -  werror (describe_backtrace (err)); +  master()->handle_error (err);    // Zap some of the remaining caches.    destruct(argcache);    destruct(cache);   #ifdef THREADS -  + #if constant(Filesystem.Monitor.basic) +  stop_fsgarb(); + #endif    if (mixed err = catch (stop_handler_threads())) -  werror (describe_backtrace (err)); +  master()->handle_error (err);   #endif /* THREADS */    roxenloader.real_exit(exit_code);    }    if (shutdown_recurse++) return;      #ifndef NO_SLOW_REQ_BT    // Turn off the backend thread monitor while we're shutting down.    slow_be_timeout_changed();   #endif       if (mixed err = catch(stop_all_configurations())) -  werror (describe_backtrace (err)); +  master()->handle_error (err);      #ifdef SNMP_AGENT    if(objectp(snmpagent)) {    snmpagent->stop_trap();    snmpagent->disable();    }   #endif       call_out(really_low_shutdown, 0.1, exit_code);   }
Roxen.git/server/base_server/roxen.pike:572:   }         /*    * handle() stuff    */      #ifdef THREADS   // function handle = threaded_handle;    - Thread do_thread_create(string id, function f, mixed ... args) + Thread.Thread do_thread_create(string id, function f, mixed ... args)   {    Thread.Thread t = thread_create(f, @args);    name_thread( t, id );    return t;   }      #if 1   constant Queue = Thread.Queue;   #else   // Shamelessly uses facts about pikes preemting algorithm.
Roxen.git/server/base_server/roxen.pike:624: Inside #if defined(THREADS)
   // above. (Must have an extra ref to the mutex since the    // MutexKey doesn't keep one.)    Thread.Mutex m = Thread.Mutex();    r_cond::wait (m->lock());    }    mixed tmp = buffer[r_ptr];    buffer[r_ptr++] = 0; // Throw away any references.    return tmp;    }    -  mixed tryread() +  mixed try_read()    {    if (!(w_ptr - r_ptr)) return ([])[0];    mixed tmp = buffer[r_ptr];    buffer[r_ptr++] = 0; // Throw away any references.    return tmp;    }       // Warning: This function isn't thread safe.    void write(mixed v)    {
Roxen.git/server/base_server/roxen.pike:658: Inside #if undefined(NO_SLOW_REQ_BT)
  // This is a system to dump all threads whenever a request takes   // longer than a configurable timeout.      protected Pike.Backend slow_req_monitor; // Set iff slow req bt is enabled.   protected float slow_req_timeout, slow_be_timeout;      protected void slow_req_monitor_thread (Pike.Backend my_monitor)   {    // my_monitor is just a safeguard to ensure we don't get multiple    // monitor threads. +  name_thread(this_thread(), "Slow request monitor");    while (slow_req_monitor == my_monitor) -  slow_req_monitor (3600); +  slow_req_monitor (3600.0); +  name_thread(this_thread(), 0);   }      protected mixed slow_be_call_out;      protected void slow_be_before_cb()   {   #ifdef DEBUG    if (this_thread() != backend_thread) error ("Run from wrong thread.\n");   #endif    if (Pike.Backend monitor = slow_be_call_out && slow_req_monitor) {    monitor->remove_call_out (slow_be_call_out);    slow_be_call_out = 0;    }   }      protected void slow_be_after_cb()   { -  +  // FIXME: This should try to compensate for delays due to the pike +  // gc, because it just causes noise here.   #ifdef DEBUG    if (this_thread() != backend_thread) error ("Run from wrong thread.\n");   #endif    if (slow_be_timeout > 0.0)    if (Pike.Backend monitor = slow_req_monitor)    slow_be_call_out = monitor->call_out (dump_slow_req, slow_be_timeout,    this_thread(), slow_be_timeout);   }      void slow_req_count_changed()
Roxen.git/server/base_server/roxen.pike:822:   {    return ((functionp (task[0]) ?    sprintf ("%s: %s", Function.defined (task[0]),    master()->describe_function (task[0])) :    programp (task[0]) ?    sprintf ("%s: %s", Program.defined (task[0]),    master()->describe_program (task[0])) :    sprintf ("%O", task[0])) +    "(" +    map (task[1], lambda (mixed arg) -  {return sprintf ("%O", arg);}) * ", " + +  {return RXML.utils.format_short (arg, 200);}) * ", " +    ")");   }    -  + protected mapping(Thread.Thread:int) thread_task_start_times = ([]); +  + mapping(Thread.Thread:int) get_thread_task_start_times() + { +  // Also needed in Admin interface's thread wizard +  return thread_task_start_times + ([ ]); + } +    local protected void handler_thread(int id)   //! The actual handling function. This functions read function and   //! parameters from the queue, calls it, then reads another one. There   //! is a lot of error handling to ensure that nothing serious happens if   //! the handler function throws an error.   {    THREAD_WERR("Handle thread ["+id+"] started");    mixed h, q;    set_u_and_gid (1);   #ifdef TEST_EUID_CHANGE
Roxen.git/server/base_server/roxen.pike:865:    THREAD_WERR("Handle thread ["+id+"] waiting for next event");    if(arrayp(h=handle_queue->read()) && h[0]) {    THREAD_WERR(sprintf("Handle thread [%O] calling %s",    id, debug_format_queue_task (h)));    set_locale();    busy_threads++;    thread_flagged_as_busy = 1;    handler_num_runs++;       int start_hrtime = gethrtime(); +  thread_task_start_times[this_thread()] = start_hrtime;    float handler_vtime = gauge {   #ifndef NO_SLOW_REQ_BT    if (h[0] != bg_process_queue &&    // Leave out bg_process_queue. It makes a timeout on    // every individual job instead.    (monitor = slow_req_monitor) && slow_req_timeout > 0.0) {    call_out = monitor->call_out (dump_slow_req, slow_req_timeout,    this_thread(), slow_req_timeout);    h[0](@h[1]);    monitor->remove_call_out (call_out);    }    else   #endif    {    h[0](@h[1]);    }    };    float handler_rtime = (gethrtime() - start_hrtime)/1E6; -  +  thread_task_start_times[this_thread()] = 0;       h=0;    busy_threads--;    thread_flagged_as_busy = 0;    if (handler_rtime > 0.01) handler_num_runs_001s++;    if (handler_rtime > 0.05) handler_num_runs_005s++;    if (handler_rtime > 0.15) handler_num_runs_015s++;    if (handler_rtime > 0.50) handler_num_runs_05s++;    if (handler_rtime > 1.00) handler_num_runs_1s++;    if (handler_rtime > 5.00) handler_num_runs_5s++;
Roxen.git/server/base_server/roxen.pike:933:    THREAD_WERR("Handle thread [" + id + "] released");    }    } while(1);    }) {    if (thread_flagged_as_busy)    busy_threads--;   #ifndef NO_SLOW_REQ_BT    if (call_out) monitor->remove_call_out (call_out);   #endif    if (h = catch { -  report_error(/*LOCALE("", "Uncaught error in handler thread: %s" -  "Client will not get any response from Roxen.\n"),*/ -  describe_backtrace(q)); +  master()->handle_error (q);    if (q = catch {h = 0;}) {    report_error(LOC_M(5, "Uncaught error in handler thread: %sClient "    "will not get any response from Roxen.")+"\n",    describe_backtrace(q));    catch (q = 0);    }    }) {    catch {    report_error("Error reporting error:\n");    report_error(sprintf("Raw error: %O\n", h[0]));
Roxen.git/server/base_server/roxen.pike:1039:      void release_handler_threads (int numthreads)   //! Releases any handler threads put on hold. If necessary new threads   //! are started to ensure that at least @[numthreads] threads are   //! responding. Threads that haven't arrived to the hold state since   //! @[hold_handler_threads] are considered nonresponding.   {    if (Thread.Condition cond = hold_wakeup_cond) {    // Flush out any remaining hold messages from the queue.    for (int i = handle_queue->size(); i && num_hold_messages; i--) { -  mixed task = handle_queue->tryread(); +  mixed task = handle_queue->try_read();    if (task == 1) num_hold_messages--;    else handle_queue->write (task);    }   #ifdef DEBUG    if (num_hold_messages)    error ("num_hold_messages is bogus (%d).\n", num_hold_messages);   #endif    num_hold_messages = 0;       int blocked_threads = number_of_threads - threads_on_hold;
Roxen.git/server/base_server/roxen.pike:1072:    report_notice ("Created %d new handler threads to compensate "    "for %d blocked ones.\n", threads_to_create, blocked_threads);    }    }    else {    THREAD_WERR("Ignoring request to release handler threads during stop");    return;    }   }    + //! + int handler_threads_on_hold() {return !!hold_wakeup_cond;} +    protected Thread.MutexKey backend_block_lock;      void stop_handler_threads()   //! Stop all the handler threads and the backend, but give up if it   //! takes too long.   { -  int timeout=10; +  int timeout=15; // Timeout if the bg queue doesn't get shorter. +  int background_run_timeout = 100; // Hard timeout that cuts off the bg queue.   #if constant(_reset_dmalloc)    // DMALLOC slows stuff down a bit...    timeout *= 10; -  +  background_run_timeout *= 3;   #endif /* constant(_reset_dmalloc) */ -  +     report_debug("Stopping all request handler threads.\n");       // Wake up any handler threads on hold, and ensure none gets on hold    // after this.    if (Thread.Condition cond = hold_wakeup_cond) {    hold_wakeup_cond = 0;    cond->broadcast();    }       while(number_of_threads>0) {
Roxen.git/server/base_server/roxen.pike:1111:    Thread.Mutex mutex = Thread.Mutex();    backend_block_lock = mutex->lock();    call_out (lambda () {    thread_reap_cnt--;    report_debug("Backend thread stopped.\n");    mutex->lock();    error("Backend stop failed.\n");    }, 0);    }    -  while(thread_reap_cnt) { +  int prev_bg_len = bg_queue_length(); +  +  while (thread_reap_cnt) {    sleep(0.1); -  if(--timeout<=0) { +  +  if (--timeout <= 0) { +  int cur_bg_len = bg_queue_length(); +  if (prev_bg_len < cur_bg_len) +  // Allow more time if the background queue is being worked off. +  timeout = 10; +  prev_bg_len = cur_bg_len; +  } +  +  if(--background_run_timeout <= 0 || timeout <= 0) {    report_debug("Giving up waiting on threads; " -  "%d threads blocked.\n", thread_reap_cnt); +  "%d threads blocked, %d jobs in the background queue.\n", +  thread_reap_cnt, bg_queue_length());   #ifdef DEBUG    describe_all_threads();   #endif    return;    }    }   }      #else   // handle function used when THREADS is not enabled.
Roxen.git/server/base_server/roxen.pike:1209:    // I hope that this will solve all your problems. /per    if( really > 0 )    return lambda( mixed ... args ){ call_out( f, 0, @args ); };    if( really < 0 )    return f;    return SignalAsyncVerifier( f )->call;   }      #ifdef THREADS   protected Thread.Queue bg_queue = Thread.Queue(); - protected int bg_process_running; + protected Thread.Thread bg_process_thread;      // Use a time buffer to strike a balance if the server is busy and   // always have at least one busy thread: The maximum waiting time in   // that case is somewhere between bg_time_buffer_min and   // bg_time_buffer_max. If there are only short periods of time between   // the queue runs, the max waiting time will shrink towards the   // minimum.   protected constant bg_time_buffer_max = 30;   protected constant bg_time_buffer_min = 0;   protected int bg_last_busy = 0;
Roxen.git/server/base_server/roxen.pike:1238: Inside #if defined(THREADS)
  int bg_acc_time = 0;   int bg_acc_cpu_time = 0;      int bg_queue_length()   {    return bg_queue->size();   }      protected void bg_process_queue()   { -  if (bg_process_running) return; +  if (bg_process_thread) return;    // Relying on the interpreter lock here. -  bg_process_running = 1; +  bg_process_thread = this_thread();       int maxbeats =    min (time() - bg_last_busy, bg_time_buffer_max) * (int) (1 / 0.01);      #ifndef NO_SLOW_REQ_BT    Pike.Backend monitor;    mixed call_out;   #endif       if (mixed err = catch {    // Make sure we don't run forever if background jobs are queued    // over and over again, to avoid starving other threads. (If they    // are starved enough, busy_threads will never be incremented.)    // If jobs are enqueued while running another background job,    // bg_process_queue is put on the handler queue again at the very    // end of this function. -  +  // +  // However, during shutdown we continue until the queue is really +  // empty. background_run won't queue new jobs then, and if this +  // takes too long, stop_handler_threads will exit anyway.    int jobs_to_process = bg_queue->size(); -  while (jobs_to_process-- && !shutdown_started) { +  while (hold_wakeup_cond ? jobs_to_process-- : bg_queue->size()) {    // Not a race here since only one thread is reading the queue.    array task = bg_queue->read();       // Wait a while if another thread is busy already.    if (busy_threads > 1) {    for (maxbeats = max (maxbeats, (int) (bg_time_buffer_min / 0.01));    busy_threads > 1 && maxbeats > 0;    maxbeats--)    sleep (0.01);    bg_last_busy = time();
Roxen.git/server/base_server/roxen.pike:1284: Inside #if defined(THREADS)
   bg_queue->size());   #endif       float task_vtime, task_rtime;    bg_num_runs++;      #ifndef NO_SLOW_REQ_BT    if ((monitor = slow_req_monitor) && slow_req_timeout > 0.0) {    call_out = monitor->call_out (dump_slow_req, slow_req_timeout,    this_thread(), slow_req_timeout); -  int start_hrtime = gethrtime (1); +  int start_hrtime = gethrtime(); +  thread_task_start_times[this_thread()] = start_hrtime;    task_vtime = gauge {    if (task[0]) // Ignore things that have become destructed.    // Note: BackgroundProcess.repeat assumes that there are    // exactly two refs to task[0] during the call below.    task[0] (@task[1]);    }; -  task_rtime = (gethrtime (1) - start_hrtime) / 1e9; +  task_rtime = (gethrtime() - start_hrtime) / 1e6; +  thread_task_start_times[this_thread()] = 0;    monitor->remove_call_out (call_out);    }    else   #endif    { -  int start_hrtime = gethrtime (1); +  int start_hrtime = gethrtime(); +  thread_task_start_times[this_thread()] = start_hrtime;    task_vtime = gauge {    if (task[0])    task[0] (@task[1]);    }; -  task_rtime = (gethrtime (1) - start_hrtime) / 1e9; +  task_rtime = (gethrtime() - start_hrtime) / 1e6; +  thread_task_start_times[this_thread()] = 0;    }       if (task_rtime > 0.01) bg_num_runs_001s++;    if (task_rtime > 0.05) bg_num_runs_005s++;    if (task_rtime > 0.15) bg_num_runs_015s++;    if (task_rtime > 0.50) bg_num_runs_05s++;    if (task_rtime > 1.00) bg_num_runs_1s++;    if (task_rtime > 5.00) bg_num_runs_5s++;    if (task_rtime > 15.00) bg_num_runs_15s++;    bg_acc_cpu_time += (int)(1E6*task_vtime);
Roxen.git/server/base_server/roxen.pike:1348: Inside #if defined(THREADS) and #if defined(DEBUG_BACKGROUND_RUN)
   "took %g ms cpu time and %g ms real time\n",    task_vtime * 1000, task_rtime * 1000);   #endif       if (busy_threads > 1) bg_last_busy = time();    }    }) {   #ifndef NO_SLOW_REQ_BT    if (call_out) monitor->remove_call_out (call_out);   #endif -  bg_process_running = 0; +  bg_process_thread = 0;    handle (bg_process_queue);    throw (err);    } -  bg_process_running = 0; +  bg_process_thread = 0;    if (bg_queue->size()) {    handle (bg_process_queue);    // Sleep a short while to encourage a thread switch. This is a    // kludge to avoid starving non-handler threads, since pike    // currently (7.8.503) doesn't switch threads reliably on yields.    sleep (0.001);    }   }   #endif    -  + Thread.Thread background_run_thread() + //! Returns the thread currently executing the background_run queue, + //! or 0 if it isn't being processed. + { + #ifdef THREADS +  return bg_process_thread; + #else +  // FIXME: This is not correct. Should return something nonzero when +  // called from the call out. But noone is running without threads +  // nowadays anyway. +  return 0; + #endif + } +    mixed background_run (int|float delay, function func, mixed... args)   //! Enqueue a task to run in the background in a way that makes as   //! little impact as possible on the incoming requests. No matter how   //! many tasks are queued to run in the background, only one is run at   //! a time. The tasks won't be starved regardless of server load,   //! though.   //!   //! The function @[func] will be enqueued after approximately @[delay]   //! seconds, to be called with the rest of the arguments as its   //! arguments.
Roxen.git/server/base_server/roxen.pike:1407: Inside #if defined(DEBUG_BACKGROUND_RUN)
   programp (func) ?    sprintf ("%s: %s", Program.defined (func),    master()->describe_program (func)) :    sprintf ("%O", func),    map (args, lambda (mixed arg)    {return sprintf ("%O", arg);}) * ", ",    bg_queue->size());   #endif      #ifdef THREADS -  if (!hold_wakeup_cond) +  if (!hold_wakeup_cond) {    // stop_handler_threads is running; ignore more work. -  + #ifdef DEBUG +  report_debug ("Ignoring background job queued during shutdown: %O\n", func); + #endif    return 0; -  +  }       class enqueue(function func, mixed ... args)    {    int __hash() { return hash_value(func); }    int `==(mixed gunc) { return func == gunc; }    string _sprintf() { return sprintf("background_run(%O)", func); }    mixed `()()    {    bg_queue->write (({func, args})); -  if (!bg_process_running) +  if (!bg_process_thread)    handle (bg_process_queue);    }    };       if (delay)    return call_out (enqueue(func, @args), delay);    else {    enqueue(func, @args)();    return 0;    }
Roxen.git/server/base_server/roxen.pike:1468: Inside #if defined(DEBUG)
   if (self_refs < 4)    error ("Minimum ref calculation wrong - have only %d refs.\n", self_refs);   #endif    if (stopping || (self_refs <= 4) || !func) {    stopping = 2; // Stopped.    return;    }    mixed err = catch {    func (@args);    }; -  if (err) { -  catch { -  report_error(LOC_M(66, "Uncaught error in background process:") + -  "%s\n", describe_backtrace(err)); -  }; -  } +  if (err) +  master()->handle_error (err);    background_run (period, repeat, func, args);    }       //! @decl void set_period (int|float period);    //!    //! Changes the period to @[period] seconds between calls.    //!    //! @note    //! This does not change the currently ongoing period, if any. That    //! might be remedied.
Roxen.git/server/base_server/roxen.pike:1801:    //! The protocol handler for this URL.    //! @member int "mib_version"    //! (Only SNMP). The version number for the configuration MIB    //! tree when it was last merged.    //! @endmapping       mapping(Configuration:mapping(string:mixed)) conf_data = ([]);    //! Maps the configuration objects to the data mappings in @[urls].       void ref(string name, mapping(string:mixed) data) -  //! Add a ref for the URL 'name' with the data 'data' +  //! Add a ref for the URL @[name] with the data @[data]. +  //! +  //! See @[urls] for documentation about the supported +  //! fields in @[data].    {    if(urls[name])    {    conf_data[urls[name]->conf] = urls[name] = data;    return; // only ref once per URL    }    if (!refs) path = data->path;    else if (path != (data->path || "")) path = 0;    refs++;    mu = 0;
Roxen.git/server/base_server/roxen.pike:2057:       mixed query_option( string x )    //! Query the port-option 'x' for this port.    {    return query( x );    }       string get_key()    //! Return the key used for this port (protocol:ip:portno)    { + #if 0    if (ip == "::")    return name + ":0:" + port;    else -  + #endif    return name+":"+ip+":"+port;    }       string get_url()    //! Return the port on URL form.    {    return (string) name + "://" +    (!ip ? "*" : has_value (ip, ":") ? "[" + ip + "]" : ip) +    ":" + port + "/";    }
Roxen.git/server/base_server/roxen.pike:2256:    (VAR)->add_warning (msg); \    cert_err_unbind(); \    cert_failure = 1; \    return; \    } while (0)       protected void filter_preferred_suites() {   #ifndef ALLOW_WEAK_SSL    // Filter weak and really weak cipher suites.    ctx->preferred_suites -= ({ +  SSL.Constants.SSL_rsa_with_des_cbc_sha, +  SSL.Constants.SSL_dhe_dss_with_des_cbc_sha,    SSL.Constants.SSL_rsa_export_with_rc4_40_md5, -  +  SSL.Constants.TLS_rsa_with_null_sha256,    SSL.Constants.SSL_rsa_with_null_sha,    SSL.Constants.SSL_rsa_with_null_md5,    SSL.Constants.SSL_dhe_dss_export_with_des40_cbc_sha,    SSL.Constants.SSL_null_with_null_null,    });   #endif    }    -  +  // NB: The TBS Tools.X509 API has been deprecated in Pike 8.0. + #pragma no_deprecation_warnings    void certificates_changed(Variable.Variable|void ignored,    void|int ignore_eaddrinuse)    {    int old_cert_failure = cert_failure;       string raw_keydata;    array(string) certificates = ({});    array(object) decoded_certs = ({});    Variable.Variable Certificates = getvar("ssl_cert_file");   
Roxen.git/server/base_server/roxen.pike:2444: Inside #if EXPORT
  #if EXPORT    ctx->export_mode();   #endif       if (!bound) {    bind (ignore_eaddrinuse);    if (old_cert_failure && bound)    report_notice (LOC_M(64, "TLS port %s opened.\n"), get_url());    }    } + #pragma deprecation_warnings       class CertificateListVariable    {    inherit Variable.FileList;       string doc()    {    return sprintf(::doc() + "\n",    combine_path(getcwd(), "../local"),    getcwd());
Roxen.git/server/base_server/roxen.pike:2593:   #endif    };    }    report_debug("\bDone [%.1fms]\n", (gethrtime()-st)/1000.0 );    return protocols;   }         mapping(string:program/*(Protocol)*/) protocols;    - // prot:ip:port ==> Protocol. + //! Lookup from protocol, IP number and port to + //! the corresponding open @[Protocol] port. + //! + //! @mapping + //! @member mapping(string:mapping(int:Protocol)) protocol_name + //! @mapping + //! @member mapping(int:Protocol) ip_number + //! @mapping + //! @member Protocol port_number + //! @[Protocol] object that holds this ip_number and port open. + //! @endmapping + //! @endmapping + //! @endmapping   mapping(string:mapping(string:mapping(int:Protocol))) open_ports = ([ ]);    - // url:"port" ==> Protocol. - mapping(string:mapping(string:Configuration|Protocol|string|array(Protocol))) + //! Lookup from URL string to the corresponding open @[Protocol] ports. + //! + //! Note that there are two classes of URL strings used as indices in + //! this mapping: + //! @dl + //! @item "prot://host_glob:port/path/" + //! A normalized URL string as returned by @[normalize_url()]. + //! + //! @[Protocol()->ref()] in the contained ports as been called + //! with the url. + //! + //! @item "port://host_glob:port/path/#opt1=val1;opt2=val2" + //! An URL string containing options as stored in the @tt{"URLs"@} + //! configuration variable, and expected as argument by + //! @[register_url()] and @[unregister_url()]. Also known + //! as an ourl. + //! @enddl + //! + //! In both cases the same set of data is stored: + //! @mapping + //! @member mapping(string:Configuration|Protocol|string|array(Protocol)|array(string)) url + //! @mapping + //! @member Protocol "port" + //! Representative open port for this URL. + //! @member array(Protocol) "ports" + //! Array of all open ports for this URL. + //! @member Configuration "conf" + //! Configuration that has registered the URL. + //! @member string "path" + //! Path segment of the URL. + //! @member string "host" + //! Hostname segment of the URL. + //! @member array(string) "skipped" + //! List of IP numbers not bound due to a corresponding + //! ANY port already being open. + //! @endmapping + //! @endmapping + mapping(string:mapping(string:Configuration|Protocol|string|array(Protocol)|array(string)))    urls = ([]); -  +    array sorted_urls = ({});      array(string) find_ips_for( string what )   { -  if( what == "*" || lower_case(what) == "any" ) +  if( what == "*" || lower_case(what) == "any" || has_value(what, "*") )    return ({   #if constant(__ROXEN_SUPPORTS_IPV6__)    "::",   #endif /* __ROXEN_SUPPORTS_IPV6__ */    0,    }); // ANY       if( is_ip( what ) )    return ({ what });    else if (has_prefix (what, "[") && what[-1] == ']') {
Roxen.git/server/base_server/roxen.pike:2645:    "unknown. Substituting with ANY")+"\n", what);    return 0; // FAIL   }      string normalize_url(string url, void|int port_match_form)   //! Normalizes the given url to a short form.   //!   //! If @[port_match_form] is set, it normalizes to the form that is   //! used for port matching, i.e. what   //! @[roxen.Protocol.find_configuration_for_url] expects. + //! + //! @note + //! Returns @expr{""@} for @[url]s that are incomplete.   {    if (!sizeof (url - " " - "\t")) return "";       Standards.URI ui = Standards.URI(url);    string host = ui->host;       if (lower_case (host) == "any" || host == "::")    host = "*";    else {    // Note: zone_to_ascii() can throw errors on invalid hostnames.
Roxen.git/server/base_server/roxen.pike:2698:    // "/", but not end with one.    path);    }       else {    ui->fragment = 0;    return (string) ui;    }   }    + //! Unregister an URL from a configuration. + //! + //! @seealso + //! @[register_url()]   void unregister_url(string url, Configuration conf)   {    string ourl = url; -  +  mapping(string:mixed) data = m_delete(urls, ourl); +  if (!data) return; // URL not registered.    if (!sizeof(url = normalize_url(url, 1))) return;       report_debug ("Unregister %s%s.\n", normalize_url (ourl),    conf ? sprintf (" for %O", conf->query_name()) : "");    -  if (urls[url] && (!conf || !urls[url]->conf || (urls[url]->conf == conf)) && -  urls[url]->port) -  { -  urls[ url ]->port->unref(url); -  m_delete( urls, url ); -  m_delete( urls, ourl ); +  mapping(string:mixed) shared_data = urls[url]; +  if (!shared_data) return; // Strange case, but URL not registered. +  +  int was_any_ip; +  if (!data->skipped && data->port) { +  if (!data->port->ip || (data->port->ip == "::")) { +  was_any_ip = data->port->port; +  report_debug("Unregistering ANY port: %O:%d\n", +  data->port->ip, data->port->port); +  } +  } +  +  foreach(data->ports || ({}), Protocol port) { +  shared_data->ports -= ({ port }); +  port->unref(url); +  m_delete(shared_data, "port"); +  } +  if (!sizeof(shared_data->ports || ({}))) { +  m_delete(urls, url); +  } else if (!shared_data->port) { +  shared_data->port = shared_data->ports[0]; +  }    sort_urls(); -  +  +  if (was_any_ip) { +  foreach(urls; string url; mapping(string:mixed) url_info) { +  if (!url_info->skipped || !url_info->conf || +  (url_info->port && (url_info->port->port != was_any_ip))) { +  continue;    } -  +  // Re-register the ports that may have bound to the removed ANY port. +  register_url(url, url_info->conf);    } -  +  } + }      array all_ports( )   { -  return Array.uniq( values( urls )->port )-({0}); +  // FIXME: Consider using open_ports instead. +  return Array.uniq( (values( urls )->ports - ({0})) * ({}) )-({0});   }      Protocol find_port( string name )   {    foreach( all_ports(), Protocol p )    if( p->get_key() == name )    return p;   }      void sort_urls()   {    sorted_urls = indices( urls );    sort( map( map( sorted_urls, strlen ), `-), sorted_urls );   }    -  + //! Register an URL for a configuration. + //! + //! @seealso + //! @[unregister_url()]   int register_url( string url, Configuration conf )   {    string ourl = url;    if (!sizeof (url - " " - "\t")) return 1;       Standards.URI ui = Standards.URI(url);    mapping opts = ([]);    string a, b;    foreach( (ui->fragment||"")/";", string x )    {
Roxen.git/server/base_server/roxen.pike:2758:    if( (int)opts->nobind )    {    report_warning(    LOC_M(61,"Not binding the port %O - disabled in configuration.")+"\n",    (string) ui );    return 0;    }       string display_url = normalize_url (url, 0);    url = normalize_url (url, 1); +  if (url == "") return 1;    ui = Standards.URI (url);       string protocol = ui->scheme;    string host = ui->host;    if (host == "" || !protocols[protocol]) {    report_error(LOC_M(19,"Bad URL %O for server %O.")+"\n",    ourl, conf->query_name());    }       int port = ui->port || protocols[protocol]->default_port;
Roxen.git/server/base_server/roxen.pike:2788:    else if( urls[ url ]->conf )    {    if( urls[ url ]->conf != conf )    {    report_error(LOC_M(20,    "Cannot register URL %s - "    "already registered by %s.")+"\n",    display_url, urls[ url ]->conf->name);    return 0;    } +  // FIXME: Is this correct?    urls[ url ]->port->ref(url, urls[url]); -  return 1; +     }    else    urls[ url ]->port->unref( url );    }       program prot;       if( !( prot = protocols[ protocol ] ) )    {    report_error(LOC_M(21, "Cannot register URL %s - "    "cannot find the protocol %s.")+"\n",    display_url, protocol);    return 0;    }    -  urls[ url ] = ([ "conf":conf, "path":path, "hostname": host ]); -  urls[ ourl ] = urls[url] + ([]); -  sorted_urls += ({ url }); +  // FIXME: Do we need to unref the old ports first in case of a reregister? +  urls[ ourl ] = ([ "conf":conf, "path":path, "hostname": host ]); +  if (!urls[url]) { +  urls[ url ] = urls[ourl] + ([]); +  sorted_urls += ({ url }); // FIXME: Not exactly sorted... +  }       array(string)|int(-1..0) required_hosts;       if (is_ip(host))    required_hosts = ({ host });    else if(!sizeof(required_hosts =    filter(replace(opts->ip||"", " ","")/",", is_ip)) ) {    required_hosts = find_ips_for( host );    if (!required_hosts) {    // FIXME: Used to fallback to ANY.
Roxen.git/server/base_server/roxen.pike:2835:    // always add 'ANY' (0) and 'IPv6_ANY' (::) here, as empty mappings,    // for speed reasons.    // There is now no need to check for both open_ports[prot][0] and    // open_ports[prot][0][port], we can go directly to the latter    // test.    m = open_ports[ protocol ] = ([ 0:([]), "::":([]) ]);       if (prot->supports_ipless ) {    // Check if the ANY port is already open for this port, since this    // protocol supports IP-less virtual hosting, there is no need to -  // open yet another port if it is, since that would mosts probably +  // open yet another port if it is, since that would most probably    // only conflict with the ANY port anyway. (this is true on most    // OSes, it works on Solaris, but fails on linux)    array(string) ipv6 = filter(required_hosts - ({ 0 }), has_value, ":");    array(string) ipv4 = required_hosts - ipv6;    if (m[0][port] && sizeof(ipv4 - ({ 0 }))) {    // We have a non-ANY IPv4 IP number. -  +  // Keep track of the ips in case the ANY port is removed. +  urls[ourl]->skipped = ipv4;    ipv4 = ({ 0 });    }   #if constant(__ROXEN_SUPPORTS_IPV6__)    if (m["::"][port] && sizeof(ipv6 - ({ "::" }))) {    // We have a non-ANY IPv6 IP number. -  +  // Keep track of the ips in case the ANY port is removed. +  urls[ourl]->skipped += ipv6;    ipv6 = ({ "::" });    }    required_hosts = ipv6 + ipv4;   #else    if (sizeof(ipv6)) {    foreach(ipv6, string p) {    report_warning(LOC_M(65, "Cannot open port %s for URL %s - "    "IPv6 support disabled.\n"),    p, display_url);    }
Roxen.git/server/base_server/roxen.pike:2874:       foreach(required_hosts, string required_host)    {    if( m[ required_host ] && m[ required_host ][ port ] )    {    if (required_host == "::") opened_ipv6_any_port = 1;       m[required_host][port]->ref(url, urls[url]);       urls[url]->port = m[required_host][port]; +  if (urls[url]->ports) { +  urls[url]->ports += ({ m[required_host][port] }); +  } else { +  urls[url]->ports = ({ m[required_host][port] }); +  }    urls[ourl]->port = m[required_host][port];    if (urls[ourl]->ports) {    urls[ourl]->ports += ({ m[required_host][port] });    } else {    urls[ourl]->ports = ({ m[required_host][port] });    }    continue; /* No need to open a new port */    }       if( !m[ required_host ] )    m[ required_host ] = ([ ]);          Protocol prot_obj;    if (mixed err = catch {    prot_obj = m[ required_host ][ port ] =    prot( port, required_host,    // Don't complain if binding IPv4 ANY fails with    // EADDRINUSE after we've bound IPv6 ANY. -  // Most systems seems to bind booth IPv4 ANY and +  // Most systems seems to bind both IPv4 ANY and    // IPv6 ANY for "::"    !required_host && opened_ipv6_any_port);    }) {    failures++;   #if 0    if (has_prefix(describe_error(err), "Invalid address") &&    required_host && has_value(required_host, ":")) {    report_error(sprintf("Failed to initialize IPv6 port for URL %s"    " (ip %s).\n",    display_url, required_host));
Roxen.git/server/base_server/roxen.pike:2929:    if (required_host == "::") opened_ipv6_any_port = 1;       if (prot_obj->bound == -1) {    // Got EADDRINUSE for the IPv6 case - see above. Just forget    // about this one.    m_delete (m[required_host], port);    continue;    }       urls[ url ]->port = prot_obj; +  if (urls[url]->ports) { +  urls[url]->ports += ({ prot_obj }); +  } else { +  urls[url]->ports = ({ prot_obj }); +  }    urls[ ourl ]->port = prot_obj;    if (urls[ourl]->ports) {    urls[ourl]->ports += ({ prot_obj });    } else {    urls[ourl]->ports = ({ prot_obj });    }    prot_obj->ref(url, urls[url]);       if( !prot_obj->bound )    failures++;
Roxen.git/server/base_server/roxen.pike:3162: Inside #if undefined(__NT__)
   report_debug("Anti-Block System Disabled.\n");    return;    }    report_debug("**** %s: ABS engaged!\n"    "Waited more than %d minute(s).\n",    ctime(time()) - "\n",    query("abs_timeout"));    // Paranoia exit in case describe_all_threads below hangs.    signal(signum("SIGALRM"), low_engage_abs);    int t = alarm(20); +  report_debug("\nTrying to dump backlog: \n"); +  if (mixed err = catch { +  // Catch for paranoia reasons. +  describe_all_threads(); +  }) +  master()->handle_error (err);   #ifdef THREADS -  report_debug("Handler queue:\n"); +  report_debug("\nHandler queue:\n");    if (mixed err = catch { -  +  t = alarm(20); // Restart the timeout timer.    array(mixed) queue = handle_queue->buffer[handle_queue->r_ptr..];    foreach(queue, mixed v) {    if (!v) continue;    if (!arrayp(v)) {    report_debug(" *** Strange entry: %O ***\n", v);    } else {    report_debug(" %{%O, %}\n", v/({}));    }    }    }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);   #endif -  report_debug("Trying to dump backlog: \n"); +  report_debug("\nPending call_outs:\n");    if (mixed err = catch { -  // Catch for paranoia reasons. -  describe_all_threads(); +  t = alarm(20); // Restart the timeout timer. +  foreach(call_out_info(), array info) { +  report_debug(" %4d seconds: %O(%{%O, %})\n", +  info[0], info[2], info[3]); +  }    }) -  werror (describe_backtrace (err)); +  master()->handle_error(err);    low_engage_abs();   }      void restart_if_stuck (int force)   //! @note   //! Must be called from the backend thread due to Linux peculiarities.   {    remove_call_out(restart_if_stuck);    if (!(query("abs_engage") || force))    return;
Roxen.git/server/base_server/roxen.pike:3226: Inside #if constant(ROXEN_MYSQL_SUPPORTS_UNICODE)
  // NOTE: We need to mark binary data as binary in case   // the Mysql character_set_connection has been   // set to anything other than "latin1".   #define MYSQL__BINARY "_binary"   #else   #define MYSQL__BINARY ""   #endif      function(string:Sql.Sql) dbm_cached_get;    + // Threshold in seconds for updating atime records Currently set to + // one day. + #define ATIME_THRESHOLD 60*60*24 +    class ImageCache   //! The image cache handles the behind-the-scenes caching and   //! regeneration features of graphics generated on the fly. Besides   //! being a cache, however, it serves a wide variety of other   //! interesting image conversion/manipulation functions as well.   {   #define QUERY(X,Y...) get_db()->query(X,Y)    string name;    string dir;    function draw_function;
Roxen.git/server/base_server/roxen.pike:3841:    if( array item = meta_cache[ id ] )    {    item[ 1 ] = time(1); // Update cached atime.    return item[ 0 ];    }      #ifdef ARG_CACHE_DEBUG    werror("restore meta %O\n", id );   #endif    array(mapping(string:string)) q = -  QUERY("SELECT meta FROM "+name+" WHERE id=%s", id ); +  QUERY("SELECT meta, UNIX_TIMESTAMP() - atime as atime_diff " +  "FROM "+name+" WHERE id=%s", id );       string s;    if(!sizeof(q) || !strlen(s = q[0]->meta))    return 0;       mapping m;    if (catch (m = decode_value (s)))    {    report_error( "Corrupt data in cache-entry "+id+".\n" );    QUERY( "DELETE FROM "+name+" WHERE id=%s", id);    return 0;    }    -  QUERY("UPDATE "+name+" SET atime=UNIX_TIMESTAMP() WHERE id=%s",id ); +  // Update atime only if older than threshold. +  if((int)q[0]->atime_diff > ATIME_THRESHOLD) { +  QUERY("UPDATE LOW_PRIORITY "+name+" " +  " SET atime = UNIX_TIMESTAMP() " +  " WHERE id = %s ", id); +  }    return meta_cache_insert( id, m );    }       protected void sync_meta()    { -  +  mapping tmp = meta_cache; +  meta_cache = ([]);    // Sync cached atimes. -  foreach(meta_cache; string id; array value) { +  foreach(tmp; string id; array value) {    if (value[1])    QUERY("UPDATE "+name+" SET atime=%d WHERE id=%s",    value[1], id);    } -  meta_cache = ([]); +     }       void flush(int|void age)    //! Flush the cache. If an age (an integer as returned by    //! @[time()]) is provided, only images with their latest access before    //! that time are flushed.    {    int num; - #ifdef DEBUG + #if defined(DEBUG) || defined(IMG_CACHE_DEBUG)    int t = gethrtime();    report_debug("Cleaning "+name+" image cache ... ");   #endif    sync_meta();    uid_cache = ([]);    rst_cache = ([]);    if( !age )    { - #ifdef DEBUG + #if defined(DEBUG) || defined(IMG_CACHE_DEBUG)    report_debug("cleared\n");   #endif    QUERY( "DELETE FROM "+name );    num = -1;    return;    }       array(string) ids =    QUERY( "SELECT id FROM "+name+" WHERE atime < "+age)->id;       num = sizeof( ids );       int q;    while(q<sizeof(ids)) {    string list = map(ids[q..q+99], get_db()->quote) * "','";    q+=100; -  QUERY( "DELETE FROM "+name+" WHERE id in ('"+list+"')" ); +  QUERY( "DELETE LOW_PRIORITY FROM "+name+" WHERE id in ('"+list+"')" );    }      #if 0    // Disabled. This can take a significant amount of time to run,    // and we really can't afford an unresponsive image cache - it can    // easily hang all handler threads. Besides, it's doubtful if this    // is of any use since the space for the deleted records probably    // will get reused soon enough anyway. /mast    if( num )    catch    {    // Old versions of Mysql lacks OPTIMIZE. Not that we support    // them, really, but it might be nice not to throw an error, at    // least. - #ifdef DEBUG + #if defined(DEBUG) || defined(IMG_CACHE_DEBUG)    report_debug("Optimizing database ... ", name);   #endif    QUERY( "OPTIMIZE TABLE "+name );    };   #endif    - #ifdef DEBUG + #if defined(DEBUG) || defined(IMG_CACHE_DEBUG)    report_debug("%s removed (%dms)\n",    (num==-1?"all":num?(string)num:"none"),    (gethrtime()-t)/1000);   #endif    }       array(int) status(int|void age)    //! Return the total number of images in the cache, their cumulative    //! sizes in bytes and, if an age time_t was supplied, the number of    //! images that has not been accessed after that time is returned
Roxen.git/server/base_server/roxen.pike:4066: Inside #if defined(ARG_CACHE_DEBUG)
  #ifdef ARG_CACHE_DEBUG    werror("data: %O id: %O\n", na, id );   #endif    if(! (res=restore( na,id )) )    {    mixed err;    if (nodraw || (err = catch {    if (mapping res = draw( na, id ))    return res;    })) { -  // File not found. -  -  if(arrayp(err) && sizeof(err) && stringp(err[0])) -  { +  if (objectp (err) && err->is_RXML_Backtrace && !RXML_CONTEXT) { +  // If we get an rxml error and there's no rxml context then +  // we're called from a direct request to the image cache. +  // The error ought to have been reported in the page that +  // generated the link to the image cache, but since it's too +  // late for that now, we just log it as a (brief) server +  // error with the referring page. +  string errmsg = "Error in " + name + " image generation: " + +  err->msg; +  if (sizeof (id->referer)) +  errmsg += " Referrer: " + id->referer[0]; +  report_error (errmsg + "\n"); +  return 0; +  } else if (arrayp(err) && sizeof(err) && stringp(err[0])) {    if (sscanf(err[0], "Requesting unknown key %s\n",    string message) == 1)    { -  +  // File not found.    report_debug("Requesting unknown key %s %O from %O\n",    message,    id->not_query,    (sizeof(id->referer)?id->referer[0]:"unknown page"));    return 0;    } -  if (sscanf(err[0], "Failed to load specified image [\"%s\"]\n", -  string message) == 1) -  { -  report_debug("Failed to load specified image %O from %O - referrer %O\n", -  message, -  id->not_query, -  (sizeof(id->referer)?id->referer[0]:"unknown page")); -  return 0; +     } -  +  throw (err);    } -  report_debug("Error in draw: %s\n", describe_backtrace(err)); -  return 0; -  } +     if( !(res = restore( na,id )) ) {    error("Draw callback %O did not generate any data.\n"    "na: %O\n"    "id: %O\n",    draw_function, na, id);    }    }    res->stat = ({ 0, 0, 0, 900000000, 0, 0, 0, 0, 0 });       // Setting the cacheable flag is done in order to get headers sent which
Roxen.git/server/base_server/roxen.pike:4194:    error("Expected mapping as the first element of the argument array\n");    update_args( data[0] );    ci = map( map( data, tomapp ), argcache->store, timeout )*"$";    } else    ci = data;    update_args = 0; // To avoid garbage.       if( zero_type( uid_cache[ ci ] ) )    {    uid_cache[ci] = user; -  if( catch(QUERY("INSERT INTO "+name+" " -  "(id,uid,atime) VALUES (%s,%s,UNIX_TIMESTAMP())", -  ci, user||"")) ) -  QUERY( "UPDATE "+name+" SET uid=%s WHERE id=%s", -  user||"", ci ); +  // Make sure to only update the entry if it does not already +  // exists or has wrong uid. Allways updating the table will +  // casue mysql to lock the table and cause a potential gobal +  // ImageCache stall. +  string uid = user || ""; +  array q = QUERY("SELECT uid from "+name+" where id=%s", ci); +  if(!sizeof(q) || (sizeof(q) && q[0]->uid != uid)) { +  QUERY("INSERT INTO "+name+" " +  "(id,uid,atime) VALUES (%s,%s,UNIX_TIMESTAMP()) " +  "ON DUPLICATE KEY UPDATE uid=%s", +  ci, uid, uid);    } -  +  }      #ifndef NO_ARG_CACHE_SB_REPLICATE    if(id->misc->persistent_cache_crawler || id->misc->do_replicate_argcache) {    // Force an update of rep_time for the requested arg cache id.    foreach(ci/"$", string key) {   #if ARGCACHE_DEBUG    werror("Request for id %O from prefetch crawler.\n", key);   #endif /* ARGCACHE_DEBUG */    argcache->refresh_arg(key);    }
Roxen.git/server/base_server/roxen.pike:4243:    master()->resolv("DBManager.is_module_table")    ( 0,"local",name,"Image cache for "+name);       QUERY("CREATE TABLE "+name+" ("    "id CHAR(64) NOT NULL PRIMARY KEY, "    "size INT UNSIGNED NOT NULL DEFAULT 0, "    "uid CHAR(32) NOT NULL DEFAULT '', "    "atime INT UNSIGNED NOT NULL DEFAULT 0,"    "meta MEDIUMBLOB NOT NULL DEFAULT '',"    "data MEDIUMBLOB NOT NULL DEFAULT ''," -  "INDEX atime (atime)" +  "INDEX atime_id (atime, id)"    ")" );    } -  +  +  // Create index in old databases. Index is used when flushing old +  // entries. Column 'id' is included in index in order to avoid +  // reading data file. +  array(mapping(string:mixed)) res = QUERY("SHOW INDEX FROM " + name); +  if(search(res->Key_name, "atime_id") < 0) { +  report_debug("Updating " + name + " image cache: " +  "Adding index atime_id on %s... ", name); +  int start_time = gethrtime(); +  QUERY("CREATE INDEX atime_id ON " + name + " (atime, id)"); +  report_debug("complete. [%f s]\n", (gethrtime() - start_time)/1000000.0); +  report_debug("Updating " + name + " image cache: " +  "Dropping index atime on %s... ", name); +  start_time = gethrtime(); +  QUERY("DROP INDEX atime ON " + name); +  report_debug("complete. [%f s]\n", (gethrtime() - start_time)/1000000.0);    } -  +  }       Sql.Sql get_db()    {    return dbm_cached_get("local");    }       protected void init_db( )    {    catch(sync_meta());    setup_tables();
Roxen.git/server/base_server/roxen.pike:4284:       void create( string id, function draw_func )    //! Instantiate an image cache of your own, whose image files will    //! be stored in a table `id' in the cache mysql database,    //!    //! The `draw_func' callback passed will be responsible for    //! (re)generation of the images in the cache. Your draw callback    //! may take any arguments you want, depending on the first argument    //! you give the <ref>store()</ref> method, but its final argument    //! will be the RequestID object. +  //! +  //! @note +  //! Use @[RXML.run_error] or @[RXML.parse_error] within the draw +  //! function to throw user level drawing errors, e.g. invalid or +  //! missing images or argument errors. If it's called within a +  //! graphics tag then the error is thrown directly and reported +  //! properly by the rxml evaluator. If it's called later, i.e. in a +  //! direct request to the image cache, then it is catched by the +  //! @[ImageCache] functions and reported in as good way as possible, +  //! i.e. currently briefly in the debug log.    {    name = id;    draw_function = draw_func;    init_db();    // Support that the 'local' database moves.    master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db );       // Always remove entries that are older than one week.    background_run( 10, do_cleanup );    }       void destroy()    {    if (mixed err = catch(sync_meta())) {    report_warning("Failed to sync cached atimes for "+name+"\n");   #if 0   #ifdef DEBUG -  report_debug (describe_backtrace (err)); +  master()->handle_error (err);   #endif   #endif    }    }   }         class ArgCache   //! Generic cache for storing away a persistent mapping of data to be   //! refetched later by a short string key. This being a cache, your   //! data may be thrown away at random when the cache is full.   { -  + #define GET_DB() \ +  Sql.Sql db = dbm_cached_get("local")   #undef QUERY   #define QUERY(X,Y...) db->query(X,Y) -  Sql.Sql db; +     string name;      #define CACHE_SIZE 900    -  Thread.Mutex mutex = Thread.Mutex(); -  // Allow recursive locks, since it's normal here. - # define LOCK() mixed __ = mutex->lock (2) -  +    #ifdef ARGCACHE_DEBUG   #define dwerror(ARGS...) werror(ARGS)   #else   #define dwerror(ARGS...) 0   #endif       //! Cache of the latest entries requested or stored.    //! Limited to @[CACHE_SIZE] (currently @expr{900@}) entries.    protected mapping(string|int:mixed) cache = ([ ]);       //! Cache of cache-ids that have no expiration time.    //! This cache is maintained in sync with @[cache].    //! Note that entries not in this cache may still have    //! unlimited expiration time.    protected mapping(string|int:int) no_expiry = ([ ]);       protected void setup_table()    { -  +  GET_DB(); +     // New style argument2 table.    if(catch(QUERY("SELECT id FROM "+name+"2 LIMIT 0")))    {    master()->resolv("DBManager.is_module_table")    ( 0, "local", name+"2",    "The argument cache, used to map between "    "a unique string and an argument mapping" );    catch(QUERY("DROP TABLE "+name+"2" ));    QUERY("CREATE TABLE "+name+"2 ("    "id CHAR(32) PRIMARY KEY, "    "ctime DATETIME NOT NULL, "    "atime DATETIME NOT NULL, "    "rep_time DATETIME NOT NULL, " -  +  "sync_time INT NULL, "    "timeout INT NULL, "    "contents MEDIUMBLOB NOT NULL, " -  " INDEX(timeout))"); +  " INDEX(timeout)," +  " INDEX(sync_time)" +  ")");    }       if (catch (QUERY ("SELECT rep_time FROM " + name + "2 LIMIT 0")))    {    // Upgrade a table without rep_time.    QUERY ("ALTER TABLE " + name + "2"    " ADD rep_time DATETIME NOT NULL"    " AFTER atime");    }       if (catch (QUERY ("SELECT timeout FROM " + name + "2 LIMIT 0")))    {    // Upgrade a table without timeout.    QUERY ("ALTER TABLE " + name + "2 "    " ADD timeout INT NULL "    "AFTER rep_time");    QUERY ("ALTER TABLE " + name + "2 "    " ADD INDEX(timeout)");    }    -  +  if (catch (QUERY ("SELECT sync_time FROM " + name + "2 LIMIT 0"))) +  { +  // Upgrade a table without sync_time. +  QUERY ("ALTER TABLE " + name + "2" +  " ADD sync_time INT NULL" +  " AFTER rep_time"); +  QUERY ("ALTER TABLE " + name + "2 " +  " ADD INDEX(sync_time)"); +  } +     catch {    array(mapping(string:mixed)) res =    QUERY("DESCRIBE "+name+"2 contents");       if(res[0]->Type == "blob") {    QUERY("ALTER TABLE "+name+"2 MODIFY contents MEDIUMBLOB NOT NULL");    werror("ArgCache: Extending \"contents\" field in table \"%s2\" from BLOB to MEDIUMBLOB.\n", name);    }    };    }       protected void do_cleanup()    { -  +  GET_DB();    QUERY("DELETE LOW_PRIORITY FROM " + name + "2 "    " WHERE timeout IS NOT NULL "    " AND timeout < %d", time());    }       protected void cleanup_process( )    {    // Flushes may be costly in large sites (since there's no index    // on the timeout field) so schedule next run sometime after    // 04:30 the day after tomorrow.
Roxen.git/server/base_server/roxen.pike:4416:       do_cleanup();    }       protected void init_db()    {    // Delay DBManager resolving to before the 'roxen' object is    // compiled.    cache = ([]);    no_expiry = ([]); -  db = dbm_cached_get("local"); +     setup_table( );       // Cleanup exprired entries on start.    background_run( 10, cleanup_process );    }       protected void create( string _name )    {    name = _name;    init_db();    // Support that the 'local' database moves (not really nessesary,    // but it won't hurt either)    master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db );    get_plugins();    }       protected string read_encoded_args( string id, int dont_update_atime )    { -  LOCK(); +  GET_DB();    array res = QUERY("SELECT contents FROM "+name+"2 "    " WHERE id = %s", id);    if(!sizeof(res))    return 0;    if (!dont_update_atime)    QUERY("UPDATE LOW_PRIORITY "+name+"2 "    " SET atime = NOW() "    " WHERE id = %s", id);    return res[0]->contents;    }       // Callback used in replicate.pike    void create_key( string id, string encoded_args, int|void timeout )    {    if (!zero_type(timeout) && (timeout < time(1))) return; // Expired. -  LOCK(); +  GET_DB();    array(mapping) rows = -  QUERY("SELECT id, contents FROM "+name+"2 WHERE id = %s", id ); +  QUERY("SELECT id, contents, timeout, " +  "UNIX_TIMESTAMP() - UNIX_TIMESTAMP(atime) as atime_diff " +  "FROM "+name+"2 " +  "WHERE id = %s", id ); +     foreach( rows, mapping row )    if( row->contents != encoded_args ) { -  report_error("ArgCache.create_key(): " -  "Duplicate key found! Please report this to support@roxen.com: " -  "id: %O, old data: %O, new data: %O\n", +  report_error("ArgCache.create_key(): Duplicate key found! " +  "Please report this to support@roxen.com:\n" +  " id: %O\n" +  " old data: %O\n" +  " new data: %O\n" +  " Updating local database with new value.\n",    id, row->contents, encoded_args); -  error("ArgCache.create_key() Duplicate key found!\n"); +  +  // Remove the old entry (probably corrupt). No need to update +  // the database since the query below uses REPLACE INTO. +  rows = ({});    }       if(sizeof(rows)) { -  if (zero_type(timeout)) { +  // Update atime only if older than threshold. +  if((int)rows[0]->atime_diff > ATIME_THRESHOLD) {    QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET atime = NOW(), timeout = NULL " +  " SET atime = NOW() "    " WHERE id = %s", id); -  } else { +  } +  +  // Increase timeout when needed. +  if (rows[0]->timeout) { +  if (zero_type(timeout)) { +  // No timeout, i.e. infinite timeout.    QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET atime = NOW() " +  " SET timeout = NULL "    " WHERE id = %s", id); -  +  } else if (timeout > (int)rows[0]->timeout) {    QUERY("UPDATE LOW_PRIORITY "+name+"2 "    " SET timeout = %d " -  " WHERE id = %s " -  " AND timeout IS NOT NULL " -  " AND timeout < %d", -  timeout, id, timeout); +  " WHERE id = %s", timeout, id);    } -  +  }    return;    }    -  QUERY( "INSERT INTO "+name+"2 " -  "(id, contents, ctime, atime) VALUES " -  "(%s, " MYSQL__BINARY "%s, NOW(), NOW())", id, encoded_args ); +  string timeout_sql = zero_type(timeout) ? "NULL" : (string)timeout; +  // Use REPLACE INTO to cope with entries created by other threads +  // as well as corrupted entries that should be overwritten. +  QUERY( "REPLACE INTO "+name+"2 " +  "(id, contents, ctime, atime, timeout) VALUES " +  "(%s, " MYSQL__BINARY "%s, NOW(), NOW(), "+timeout_sql+")", +  id, encoded_args );    -  if (!zero_type(timeout)) { -  QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET timeout = %d " -  " WHERE id = %s", -  timeout, id); -  } -  +     dwerror("ArgCache: Create new key %O\n", id);       (plugins->create_key-({0}))( id, encoded_args );    }       protected array plugins;    protected void get_plugins()    {    plugins = ({});    foreach( ({ "../local/arg_cache_plugins", "arg_cache_plugins" }), string d)
Roxen.git/server/base_server/roxen.pike:4536:    //! @param timeout    //! Timeout for the entry in seconds from now. If @expr{UNDEFINED@},    //! the entry will not expire.    {    if (!zero_type(timeout)) timeout += time(1);    string encoded_args = encode_value_canonic( args );    string id = Gmp.mpz(Crypto.SHA1.hash(encoded_args), 256)->digits(36);    if( cache[ id ] ) {    if (!no_expiry[id]) {    // The cache id may have a timeout. +  GET_DB();    if (zero_type(timeout)) {    // No timeout now, but there may have been one earlier.    QUERY("UPDATE LOW_PRIORITY "+name+"2 "    " SET timeout = NULL "    " WHERE id = %s "    " AND timeout IS NOT NULL", id);    no_expiry[id] = 1;    } else {    // Attempt to bump the timeout.    QUERY("UPDATE LOW_PRIORITY "+name+"2 "
Roxen.git/server/base_server/roxen.pike:4593:    cache = ([]);    no_expiry = ([]);    }    cache[id] = args + ([]);    return args;    }       void delete( string id )    //! Remove the data element stored under the key @[id].    { -  LOCK(); +  GET_DB();    (plugins->delete-({0}))( id );    m_delete( cache, id );       QUERY( "DELETE FROM "+name+"2 WHERE id = %s", id );    }       int key_exists( string id )    //! Does the key @[id] exist in the cache? Returns 1 if it does, 0    //! if it was not present.    {
Roxen.git/server/base_server/roxen.pike:4632:    //! might not be included in the dump.    {    constant FETCH_ROWS = 10000;    int entry_count = 0;       // The server does only need to use file based argcache    // replication if the server don't participate in a replicate    // setup with a shared database.    if( !has_value((plugins->is_functional-({0}))(), 1) )    { +  GET_DB();    int cursor;    array(string) ids;    do {    // Note: No lock is held, so rows might be added between the    // SELECTs here. That can however only cause a slight overlap    // between the LIMIT windows since rows are only added and    // never removed, and read_dump doesn't mind the occasional    // duplicate entry.    //    // A lock will be necessary here if a garb is added, though.
Roxen.git/server/base_server/roxen.pike:4748:    }    if(s != "EOF")    return "Missing data in argcache file\n";    return 0;    }       void refresh_arg(string id)    //! Indicate that the entry @[id] needs to be included in the next    //! @[write_dump]. @[id] must be an existing entry.    { +  GET_DB();    QUERY("UPDATE "+name+"2 SET rep_time=NOW() WHERE id = %s", id);    }   }      mapping cached_decoders = ([]);   string decode_charset( string charset, string data )   {    // FIXME: This code is probably not thread-safe!    if( charset == "iso-8859-1" ) return data;    if( !cached_decoders[ charset ] )
Roxen.git/server/base_server/roxen.pike:4831:    // RXML.pmod correctly. :P    master()->resolv ("RXML.refs");    master()->resolv ("RXML.PXml");    master()->resolv ("RXML.PEnt");    foreach(({ "module.pmod","PEnt.pike", "PExpr.pike","PXml.pike",    "refs.pmod","utils.pmod" }), string q )    dump( "etc/modules/RXML.pmod/"+ q );    dump( "etc/modules/RXML.pmod/module.pmod" );    master()->add_dump_constant ("RXML.empty_tag_set",    master()->resolv ("RXML.empty_tag_set")); +  +  // Replace Val objects with versions extended for the rxml type system. +  Val->true = Roxen.true; +  Val->false = Roxen.false; +  Val->null = Roxen->sql_null = Roxen.null; +     // Already loaded. No delayed dump possible.    dump( "etc/roxen_master.pike" );    dump( "etc/modules/Roxen.pmod" );    dump( "base_server/config_userdb.pike" );    dump( "base_server/disk_cache.pike" );    dump( "base_server/roxen.pike" );    dump( "base_server/basic_defvar.pike" );    dump( "base_server/newdecode.pike" );    dump( "base_server/read_config.pike" );    dump( "base_server/global_variables.pike" );
Roxen.git/server/base_server/roxen.pike:4970:    if (!g) gid = pw[3];    }      #ifdef THREADS    Thread.MutexKey mutex_key;    object threads_disabled;    if (!from_handler_thread) {    // If this is necessary from every handler thread, these    // things are thread local and thus are no locks necessary.    if (mixed err = catch { mutex_key = euid_egid_lock->lock(); }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);    threads_disabled = _disable_threads();    }   #endif      #if constant(seteuid)    if (geteuid() != getuid()) seteuid (getuid());   #endif      #if constant(initgroups)    if (mixed err = catch {    initgroups(pw[0], gid);    // Doesn't always work - David.    }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);   #endif       if (query("permanent_uid")) {   #if constant(setuid)    if (g) {   # if constant(setgid)    setgid(gid);    if (getgid() != gid) {    report_error(LOC_M(25, "Failed to set gid.")+"\n");    g = 0;
Roxen.git/server/base_server/roxen.pike:5267: Inside #if defined(THREADS)
  #ifdef THREADS    if (sscanf( f, "http://%[^/]", string host ) ||    sscanf (f, "https://%[^/]", host)) {    mapping hd = ([    "User-Agent":version(),    "Host":host,    ]);    if (mixed err = catch {    data = Protocols.HTTP.get_url_data( f, 0, hd );    }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);    }   #endif    if( !data )    return 0;    }    }    id->misc->_load_image_called = 0;    if(!data) return 0;    return low_decode_image( data );   }
Roxen.git/server/base_server/roxen.pike:5299: Inside #if defined(THREADS)
  #ifdef THREADS    if (sscanf( f, "http://%[^/]", string host ) ||    sscanf (f, "https://%[^/]", host)) {    mapping hd = ([    "User-Agent":version(),    "Host":host,    ]);    if (mixed err = catch {    data = Protocols.HTTP.get_url_data( f, 0, hd );    }) -  werror (describe_backtrace (err)); +  master()->handle_error (err);    }   #endif    if( !data )    return res;    }    }    id->misc->_load_image_called = 0;    if(!data) return res;    return decode_layers( data, opt );   }
Roxen.git/server/base_server/roxen.pike:5353:   void create_pid_file(string where)   {   #ifndef __NT__    if(!where) return;   // where = replace(where, ({ "$pid", "$uid" }),   // ({ (string)getpid(), (string)getuid() }));       object privs = Privs("Deleting old pid file.");    r_rm(where);    privs = 0; -  if(mixed err = catch { +  +  mixed err; +  +  // Note: The server lock file is often created by the start script, but +  // there is a race, so this code is here for paranoia reasons. +  if (!Stdio.exist(sprintf("/var/run/roxen-server.%d.pid", getpid())) && +  !Stdio.exist(sprintf("/tmp/roxen-server.%d.pid", getpid()))) { +  // NB: The following won't work if there's a wrapper process +  // for Roxen (eg started via gdb, truss or valgrind), +  // but that shouldn't matter much, since the pid lock file +  // won't be used in that case anyway. +  privs = Privs("Creating pid lock."); +  if (catch { +  // Try /var/run/ first. +  hardlink(sprintf("/var/run/roxen-start.%d.pid", getppid()), +  sprintf("/var/run/roxen-server.%d.pid", getpid())); +  } && (err = catch { +  // And then /tmp/. +  hardlink(sprintf("/tmp/roxen-start.%d.pid", getppid()), +  sprintf("/tmp/roxen-server.%d.pid", getpid())); +  })) { +  report_debug("Cannot create the pid lock file %O: %s", +  sprintf("/tmp/roxen-server.%d.pid", getpid()), +  describe_error(err)); +  } +  privs = 0; +  } +  if(err = catch {    Stdio.write_file(where, sprintf("%d\n%d\n", getpid(), getppid()));    })    report_debug("Cannot create the pid file %O: %s",    where, describe_error (err));   #endif   }      Pipe.pipe shuffle(Stdio.File from, Stdio.File to,    Stdio.File|void to2,    function(:void)|void callback)
Roxen.git/server/base_server/roxen.pike:5413:    lambda (Thread.Thread a, Thread.Thread b) {    // Backend thread first, otherwise in id order.    if (a == backend_thread)    return 0;    else if (b == backend_thread)    return 1;    else    return a->id_number() > b->id_number();    });    -  int i; -  for(i=0; i < sizeof(threads); i++) { +  int hrnow = gethrtime(); +  foreach (threads, Thread.Thread thread) { +  string thread_descr = ""; +  if (string th_name = thread_name(thread, 1)) +  thread_descr += " - " + th_name; +  if (int start_hrtime = thread_task_start_times[thread]) +  thread_descr += sprintf (" - busy for %.3fs", +  (hrnow - start_hrtime) / 1e6);    report_debug(">> ### Thread 0x%x%s:\n", -  threads[i]->id_number(), -  threads[i] == backend_thread ? " (backend thread)" : "" -  ); +  thread->id_number(), +  thread_descr); +  // Use master()->describe_backtrace to sidestep the background +  // failure wrapper that's active in RUN_SELF_TEST.    report_debug(">> " + -  replace (describe_backtrace (threads[i]->backtrace()), +  replace (master()->describe_backtrace (thread->backtrace()),    "\n", "\n>> ") +    "\n");    }       array(array) queue = handle_queue->peek_array();    -  +  // Ignore the handle thread shutdown marker, if any. +  queue -= ({0}); +     if (!sizeof (queue))    report_debug ("###### No entries in the handler queue\n");    else {    report_debug ("###### %d entries in the handler queue:\n>>\n",    sizeof (queue));    foreach (queue; int i; array task)    report_debug (">> %d: %s\n", i,    replace (debug_format_queue_task (task), "\n", "\n>> "));    report_debug (">> \n");    }
Roxen.git/server/base_server/roxen.pike:5475:   constant cdt_poll_interval = 5; // Seconds.   constant cdt_dump_seq_interval = 60;      string cdt_directory, cdt_filename;      Thread.Thread cdt_thread;   int cdt_next_seq_dump;      void cdt_poll_file()   { +  name_thread(this_thread(), "Dump thread file monitor");    while (this && query ("dump_threads_by_file")) {    if (array(string) dir = r_get_dir (cdt_directory)) {    if (has_value (dir, cdt_filename)) {    r_rm (cdt_directory + "/" + cdt_filename);    describe_all_threads();    }    else if (time() >= cdt_next_seq_dump) {    dir = glob (cdt_filename + ".*", dir);    if (sizeof (dir)) {    string file = dir[0];
Roxen.git/server/base_server/roxen.pike:5498:    if (--count > 0) {    open (cdt_directory + "/" + cdt_filename + "." + count,    "cwt");    cdt_next_seq_dump = time (1) + cdt_dump_seq_interval;    }    }    }    }    sleep (cdt_poll_interval);    } +  name_thread(this_thread(), 0);    cdt_thread = 0;   }      void cdt_changed (Variable.Variable v)   {    if (cdt_directory && v->query() && !cdt_thread)    cdt_thread = Thread.thread_create (cdt_poll_file);   }      // ----------------------------------------
Roxen.git/server/base_server/roxen.pike:5560:   protected class GCTimestamp   {    array self_ref;    protected void create() {self_ref = ({this_object()});}    protected void destroy() {    werror ("GC runs at %s", ctime(time()));    GCTimestamp();    }   }    + protected int gc_start;    -  + protected mapping(string:int) gc_histogram = ([]); +    array argv;   int main(int argc, array tmp)   {    // __builtin.gc_parameters((["enabled": 0]));    argv = tmp;    tmp = 0;      #if 0    Thread.thread_create (lambda () {    while (1) {    sleep (10);    describe_all_threads();    }    });   #endif      #ifdef LOG_GC_TIMESTAMPS -  +  Pike.gc_parameters(([ "pre_cb": lambda() { +  gc_start = gethrtime(); +  werror("GC runs at %s", ctime(time())); +  }, +  "post_cb":lambda() { +  werror("GC done after %dus\n", +  gethrtime() - gc_start); +  }, +  "destruct_cb":lambda(object o) { +  gc_histogram[sprintf("%O", object_program(o))]++; +  werror("GC cyclic reference in %O.\n", +  o); +  }, +  "done_cb":lambda(int n) { +  if (!n) return; +  werror("GC zapped %d things.\n", n); +  mapping h = gc_histogram + ([]); +  if (!sizeof(h)) return; +  array i = indices(h); +  array v = values(h); +  sort(v, i); +  werror("GC histogram (accumulative):\n"); +  foreach(reverse(i)[..9], string p) { +  werror("GC: %s: %d\n", p, h[p]); +  } +  }, +  ])); +  if (!Pike.gc_parameters()->pre_cb) { +  // GC callbacks not available.    GCTimestamp(); -  +  }   #endif       // For RBF    catch(mkdir(getenv("VARDIR") || "../var"));       dbm_cached_get = master()->resolv( "DBManager.cached_get" );       dbm_cached_get( "local" )->    query( "CREATE TABLE IF NOT EXISTS "    "compiled_formats ("
Roxen.git/server/base_server/roxen.pike:5724:    name_thread( backend_thread, "Backend" );   #else    report_debug("\n"    "WARNING: Threads not enabled!\n"    "\n");   #endif /* THREADS */       foreach(({ "testca.pem", "demo_certificate.pem" }), string file_name) {    if (sizeof(roxenloader.package_directories) &&    (lfile_path(file_name) == file_name)) { -  file_name = roxenloader.package_directories[-1] + "/" + file_name; -  report_notice("Generating a new certificate: %O...\n", file_name); +  file_name = roxen_path (roxenloader.package_directories[-1] + "/" + +  file_name); +  report_notice("Generating a new certificate %s...\n", file_name);    string cert = Roxen.generate_self_signed_certificate("*");       // Note: set_u_and_gid() hasn't been called yet,    // so there's no need for Privs.    Stdio.File file = Stdio.File();    if (!file->open(file_name, "wxc", 0600)) { -  report_error("Couldn't create certificate file %O.\n", file_name); +  report_error("Couldn't create certificate file %s: %s\n", file_name, +  strerror (file->errno()));    } else if (file->write(cert) != sizeof(cert)) {    rm(cert); -  report_error("Couldn't write certificate file %O.\n", file_name); +  report_error("Couldn't write certificate file %s: %s\n", file_name, +  strerror (file->errno()));    }    }    }       enable_configurations();       string pid_file = Getopt.find_option(argv, "p", "pid-file");    if (pid_file && query("permanent_uid")) rm(pid_file);       set_u_and_gid(); // Running with the right [e]uid:[e]gid from this point on.
Roxen.git/server/base_server/roxen.pike:5764:    if( Getopt.find_option( argv, 0, "no-delayed-load" ))    enable_configurations_modules();    else    foreach( configurations, Configuration c )    if( c->query( "no_delayed_load" ) )    c->enable_all_modules();   #endif // RUN_SELF_TEST      #ifdef THREADS    start_handler_threads(); + #if constant(Filesystem.Monitor.basic) +  start_fsgarb(); + #endif   #endif /* THREADS */      #ifdef TEST_EUID_CHANGE    if (test_euid_change) {    Stdio.File f = Stdio.File();    if (f->open ("rootonly", "r") && f->read())    werror ("Backend thread can read rootonly\n");    else    werror ("Backend thread can't read rootonly\n");    }
Roxen.git/server/base_server/roxen.pike:5910:   // Called from the administration interface.   string check_variable(string name, mixed value)   {    switch(name)    {   #ifndef __NT__    case "abs_engage":    if (value)    // Make sure restart_if_stuck is called from the backend thread.    call_out(restart_if_stuck, 0, 1); -  else +  else {    remove_call_out(restart_if_stuck); -  +  alarm(0); +  }    break;    case "abs_timeout":    if (value < 0) {    return "The timeout must be >= 0 minutes.";    }    break;   #endif       case "suicide_schedule":    case "suicide_engage":
Roxen.git/server/base_server/roxen.pike:6090:    }       protected string extract_user(string from)    {    array tmp;    if (!from || sizeof(tmp = from/":")<2)    return "-";    return tmp[0]; // username only, no password    }    +  protected string get_forwarded_field(RequestID id, string field) +  { +  foreach(id->misc->forwarded || ({}), array(string|int) segment) { +  if (!arrayp(segment) || sizeof(segment) < 3) continue; +  if (segment[0] != field || segment[1] != '=') continue; +  return MIME.quote(segment[2..]); +  } +  return "-"; +  } +     void log_access( function do_write, RequestID id, mapping file );       void log_event (function do_write, string facility, string action,    string resource, mapping(string:mixed) info);       protected void do_async_write( string host, string data,    string ip, function c )    {    if( c )    c( replace( data, "\4711", (host||ip) ) );
Roxen.git/server/base_server/roxen.pike:6188:    "%s" , "url_encode (resource)", 0}),    "real-full-resource": ({"%s", ("(string)(request_id->raw_url||"    " request_id->not_query)"),    "%s" , "url_encode (resource)", 0}),    "real-cs-uri-stem": ({"%s", ("(string)(request_id->not_query||"    " (request_id->raw_url && "    " (request_id->raw_url/\"?\")[0])||"    " \"-\")"),    "%s" , "url_encode (resource)", 0}),    "protocol": ({"%s", "(string)request_id->prot", 1, "\"-\"", 0}), +  "scheme": ({"%s", "(string)((request_id->port_obj && " +  "request_id->port_obj->prot_name) || \"-\")", +  1, "\"-\"", 0 }),    "response": ({"%d", "(int)(file->error || 200)", 1, "\"-\"", 0}),    "bin-response": ({"%2c", "(int)(file->error || 200)", 1, "\"\\0\\0\"", 0}),    "length": ({"%d", "(int)file->len", 1, "\"0\"", 0}),    "bin-length": ({"%4c", "(int)file->len", 1, "\"\\0\\0\\0\\0\"", 0}),    "queue-length": ({"%d", "(int) request_id->queue_length",    1, "\"-\"", 0}),    "request-time": ({"%1.4f", ("(float)(gethrtime() - "    " request_id->hrtime) /"    "1000000.0"),    1, "\"-\"", 0}),
Roxen.git/server/base_server/roxen.pike:6257:    "cache-status": ({"%s", ("sizeof(request_id->cache_status||({}))?"    "indices(request_id->cache_status)*\",\":"    "\"nocache\""),    1, "\"-\"", 0}),    "eval-status": ({"%s", ("sizeof(request_id->eval_status||({}))?"    "indices(request_id->eval_status)*\",\":"    "\"-\""),    1, "\"-\"", 0}),    "protcache-cost": ({"%d", "request_id->misc->protcache_cost",    1, "\"-\"", 0}), -  "xff": ({"%s", ("arrayp(request_id->request_headers[" -  "\"x-forwarded-for\"]) ? " -  "(request_id->request_headers[" -  "\"x-forwarded-for\"][-1] / \",\")[0] :" -  "((request_id->request_headers[" -  "\"x-forwarded-for\"] || \"-\") / \",\")[0]"), +  "forwarded": ({"%s", ("request_id->misc->forwarded ? " +  "MIME.quote(request_id->misc->forwarded *" +  " ({ ',' })) : \"-\""),    1, "\"-\"", 0 }), -  +  "xff": ({"%s", "get_forwarded_field(request_id, \"for\")", +  1, "\"-\"", 0 }),       // Used for event logging    "facility": ({"-", 0, "%s", "facility", 0}),    "action": ({"-", 0, "%s", "action", 0}),   ]);      void run_log_format( string fmt, function c, RequestID id, mapping file )   {    (compiled_log_access[ fmt ] ||    compile_log_format( fmt )->log_access) (c,id,file);
Roxen.git/server/base_server/roxen.pike:6289:   {    (compiled_log_event[ fmt ] ||    compile_log_format( fmt )->log_event) (cb, facility, action,    resource, info);   }      protected LogFormat compile_log_format( string fmt )   {    add_constant( "___LogFormat", LogFormat );    +  // Note similar code in compile_security_pattern. +     string kmd5 = md5( fmt );       object con = dbm_cached_get("local");       {    array tmp =    con->query("SELECT full,enc FROM compiled_formats WHERE md5=%s", kmd5 );       if( sizeof(tmp) && (tmp[0]->full == fmt) )    {    LogFormat lf;    if (mixed err = catch {    lf = decode_value( tmp[0]->enc, master()->Decoder() )();    }) { - // #ifdef DEBUG -  report_error("Decoding of dumped log format failed:\n%s", -  describe_backtrace(err)); - // #endif +  if (describe_error (err) != +  "Cannot decode programs encoded with other pike version.\n") +  report_warning ("Decoding of dumped log format failed " +  "(will recompile): %s", describe_backtrace(err));    }       if (lf && lf->log_access) {    // Check that it's a new style log program (old ones have log()    // instead of log_access()).    compiled_log_access[fmt] = lf->log_access;    compiled_log_event[fmt] = lf->log_event;    return lf;    }    }
Roxen.git/server/base_server/roxen.pike:6713:   //! return means that reaching this command results in immediate   //! return, only useful for 'allow'.   //!   //! 'deny' always implies a return, no futher testing is done if a   //! 'deny' match.   {    // Now, this cache is not really all that performance critical, I    // mostly coded it as a proof-of-concept, and because it was more    // fun that trying to find the bug in the image-cache at the moment.    +  // Note similar code in compile_log_format. +  if (pattern == "") +  return 0;    string kmd5 = md5( pattern );      #if !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG)    array tmp =    dbm_cached_get( "local" )    ->query("SELECT full,enc FROM compiled_formats WHERE md5=%s", kmd5 );       if( sizeof(tmp) && (tmp[0]->full == pattern) )    {    mixed err = catch {    return decode_value( tmp[0]->enc, master()->Decoder() )()->f;    };   // #ifdef DEBUG -  report_error("Decoding of dumped log format failed:\n%s", -  describe_backtrace(err)); +  if (describe_error (err) != +  "Cannot decode programs encoded with other pike version.\n") +  report_warning ("Decoding of dumped security pattern failed " +  "(will recompile):\n%s", describe_backtrace(err));   // #endif    }   #endif /* !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG) */          string code = "";    array variables = ({ " object userdb_module",    " object authmethod = id->conf",    " string realm = \"User\"",    " mapping(string:int|mapping) state = ([])",
Roxen.git/server/base_server/roxen.pike:7084:    ({ "%y", "%m", "%d", "%h", "%H" }),    ({ "[0-9][0-9][0-9][0-9]", "[0-9][0-9]",    "[0-9][0-9]", "[0-9][0-9]", "(.+)" }))+"$")->    match(filename_candidate))    {    string compress_file = combine_path(dir, filename_candidate);    Stdio.Stat stat = file_stat(compress_file);    if(!stat || time(1) < stat->mtime + 1200)    continue; // Wait at least 20 minutes before compressing log file...    werror("Compressing log file %O\n", compress_file); -  compressor_process = Process.create_process(({ compressor_program, +  compressor_process = Process.Process(({ compressor_program,    compress_file }));    return;    }    }    }       private void do_open_co() { handle(do_open); }    private void do_open(void|object mutex_key)    { -  if (!this_object()) return; // We've been destructed, return +     if (!mutex_key) mutex_key = lock->lock(); -  if (!this_object()) return; // We've been destructed, return +        mixed parent;    if (catch { parent = function_object(object_program(this_object())); } ||    !parent) {    // Our parent (aka the configuration) has been destructed.    // Time to die.    remove_call_out(do_open_co);    remove_call_out(do_close_co);    destruct();    return;
Roxen.git/server/base_server/roxen.pike:7141:    "\n");    return;    }    opened = 1;    remove_call_out(do_open_co);    call_out(do_open_co, 900);    remove_call_out(do_close_co);    call_out(do_close_co, 10.0);    }    -  private void do_close_co() { handle(do_close); } -  private void do_close() +  private void do_close_co() { handle(close); } +  +  void close()    { -  if (!this_object()) return; // We've been destructed, return +     object mutex_key = lock->lock(); -  if (!this_object()) return; // We've been destructed, return +        destruct( fd );    opened = 0;    }       private array(string) write_buf = ({}); -  private void do_the_write_co() { handle(do_the_write); } +     private void do_the_write()    { -  if (!this_object()) return; // We've been destructed, return +     object mutex_key = lock->lock(); -  if (!this_object()) return; // We've been destructed, return +        if (!opened) do_open(mutex_key);    if (!opened) return; -  mixed err = catch (fd->write(write_buf)); +  if (!sizeof (write_buf)) return; +  +  array(string) buf = write_buf; +  // Relying on the interpreter lock here. +  write_buf = ({}); +  +  mixed err = catch (fd->write(buf));    if (err)    catch {    foreach (write_buf, string str)    if (String.width (str) > 8)    werror ("Got wide string in log output: %O\n", str);    }; -  write_buf = ({}); +     remove_call_out(do_close_co);    call_out(do_close_co, 10.0); -  +     if (err)    throw (err);    }       int write( string what )    { -  if (!this_object()) return 0; // We've been destructed, return -  object mutex_key = lock->lock(); -  if (!this_object()) return 0; // We've been destructed, return -  -  if (!sizeof(write_buf)) -  call_out(do_the_write_co, 1); +     write_buf += ({ what }); -  +  if (sizeof (write_buf) == 1) +  background_run (1, do_the_write);    return strlen(what);    }   }