Roxen.git / server / base_server / roxen.pike

version» Context lines:

Roxen.git/server/base_server/roxen.pike:1:   // This file is part of Roxen WebServer.   // Copyright © 1996 - 2009, Roxen IS.   //   // The Roxen WebServer main program.   //   // Per Hedbor, Henrik Grubbström, Pontus Hagland, David Hedbor and others.   // ABS and suicide systems contributed freely by Francesco Chemolli    - constant cvs_version="$Id: roxen.pike,v 1.1113 2012/02/27 23:32:11 mast Exp $"; + constant cvs_version="$Id$";      //! @appears roxen   //!   //! The Roxen WebServer main program.      // The argument cache. Used by the image cache.   ArgCache argcache;      // Some headerfiles   #define IN_ROXEN
Roxen.git/server/base_server/roxen.pike:26:   // Inherits   inherit "global_variables";   #ifdef SNMP_AGENT   inherit "snmpagent";   #endif   #ifdef SMTP_RELAY   inherit "smtprelay";   #endif   inherit "hosts";   inherit "disk_cache"; + inherit "fsgc";   // inherit "language";   inherit "supports";   inherit "module_support";   inherit "config_userdb";      // Used to find out which thread is the backend thread.   Thread.Thread backend_thread;      // --- Locale defines ---      //<locale-token project="roxen_start"> LOC_S </locale-token>   //<locale-token project="roxen_message"> LOC_M </locale-token> - #define LOC_S(X,Y) _STR_LOCALE("roxen_start",X,Y) - #define LOC_M(X,Y) _STR_LOCALE("roxen_message",X,Y) - #define CALL_M(X,Y) _LOCALE_FUN("roxen_message",X,Y) + //<locale-token project="roxen_config"> LOC_C </locale-token> + #define LOC_S(X,Y) _STR_LOCALE("roxen_start",X,Y) + #define LOC_M(X,Y) _STR_LOCALE("roxen_message",X,Y) + #define LOC_C(X,Y) _STR_LOCALE("roxen_config",X,Y) + #define CALL_M(X,Y) _LOCALE_FUN("roxen_message",X,Y)      // --- Debug defines ---      #ifdef SSL3_DEBUG   # define SSL3_WERR(X) report_debug("TLS port %s: %s\n", get_url(), X)   #else   # define SSL3_WERR(X)   #endif      #ifdef THREAD_DEBUG   # define THREAD_WERR(X) report_debug("Thread: "+X+"\n")   #else   # define THREAD_WERR(X)   #endif    -  + #ifdef LOG_GC_VERBOSE + #define LOG_GC_HISTOGRAM + #endif +  + #ifdef LOG_GC_HISTOGRAM + #define LOG_GC_TIMESTAMPS + #endif +    // Needed to get core dumps of seteuid()'ed processes on Linux.   #if constant(System.dumpable)   #define enable_coredumps(X) System.dumpable(X)   #else   #define enable_coredumps(X)   #endif      #define DDUMP(X) sol( combine_path( __FILE__, "../../" + X ), dump )   protected function sol = master()->set_on_load;   
Roxen.git/server/base_server/roxen.pike:107:       else if( !fname ) {    fname = master()->program_name( o );    if (!fname)    return ({0, 0});    }       string cwd = getcwd() + "/";    if (has_prefix (fname, cwd))    fname = fname[sizeof (cwd)..]; -  else if (has_prefix (fname, roxenloader.server_dir)) -  fname = fname[sizeof (roxenloader.server_dir)..]; +  else if (has_prefix (fname, roxenloader.server_dir + "/")) +  fname = fname[sizeof (roxenloader.server_dir + "/")..];       return ({fname, line});   }      string filename( program|object o )   {    [string fname, int line] = filename_2 (o);    return fname || "(unknown program)";   }      protected int once_mode;      // Note that 2.5 is a nonexisting version. It's only used for the   // cache static optimization for tags such as <if> and <emit> inside   // <cache> since that optimization can give tricky incompatibilities   // with 2.4. -  + // Note also that 5.3 only existed in the Print repository, and + // thus is skipped here.   array(string) compat_levels = ({"2.1", "2.2", "2.4", "2.5",    "3.3", "3.4",    "4.0", "4.5", -  "5.0", "5.1", "5.2" }); +  "5.0", "5.1", "5.2", "5.4", "5.5", +  "6.0", "6.1", + });      #ifdef THREADS   mapping(string:string) thread_names = ([]); -  +  + string thread_name_from_addr(string hex_addr) + { +  // Lookup using a key like "Thread.Thread(0x...)" that matches what +  // sprint("%O") generates. +  string th_key = "Thread.Thread(" + hex_addr + ")"; +  return thread_names[th_key]; + } +    string thread_name( object thread, int|void skip_auto_name )   {    string tn;    if( thread_names[ tn=sprintf("%O",thread) ] || skip_auto_name )    return thread_names[tn];    return tn;   }      void name_thread( object thread, string name )   {    string th_key = sprintf("%O", thread);    if (name)    thread_names[th_key] = name;    else    m_delete(thread_names, th_key);   }    - // This mutex is used by Privs - Thread.Mutex euid_egid_lock = Thread.Mutex(); +    #endif /* THREADS */    - /* -  * The privilege changer. Works like a mutex lock, but changes the UID/GID -  * while held. Blocks all threads. -  * -  * Based on privs.pike,v 1.36. -  */ - int privs_level; -  - protected class Privs - { - #if efun(seteuid) -  -  int saved_uid; -  int saved_gid; -  -  int new_uid; -  int new_gid; -  - #define LOGP (variables && variables->audit && variables->audit->query()) -  - #if constant(geteuid) && constant(getegid) && constant(seteuid) && constant(setegid) - #define HAVE_EFFECTIVE_USER - #endif -  -  private string _getcwd() -  { -  if (catch{return(getcwd());}) { -  return("Unknown directory (no x-bit on current directory?)"); -  } -  } -  -  private string dbt(array t) -  { -  if(!arrayp(t) || (sizeof(t)<2)) return ""; -  return (((t[0]||"Unknown program")-(_getcwd()+"/"))-"base_server/")+":"+t[1]+"\n"; -  } -  - #ifdef THREADS -  protected mixed mutex_key; // Only one thread may modify the euid/egid at a time. -  protected object threads_disabled; - #endif /* THREADS */ -  -  int p_level; -  -  void create(string reason, int|string|void uid, int|string|void gid) -  { -  // No need for Privs if the uid has been changed permanently. -  if(getuid()) return; -  - #ifdef PRIVS_DEBUG -  report_debug(sprintf("Privs(%O, %O, %O)\n" -  "privs_level: %O\n", -  reason, uid, gid, privs_level)); - #endif /* PRIVS_DEBUG */ -  - #ifdef HAVE_EFFECTIVE_USER -  array u; -  - #ifdef THREADS -  if (euid_egid_lock) { -  if (mixed err = catch { mutex_key = euid_egid_lock->lock(); }) -  master()->handle_error (err); -  } -  threads_disabled = _disable_threads(); - #endif /* THREADS */ -  -  p_level = privs_level++; -  -  /* Needs to be here since root-priviliges may be needed to -  * use getpw{uid,nam}. -  */ -  saved_uid = geteuid(); -  saved_gid = getegid(); -  seteuid(0); -  -  /* A string of digits? */ -  if(stringp(uid) && (replace(uid,"0123456789"/"",({""})*10)=="")) -  uid = (int)uid; -  -  if(stringp(gid) && (replace(gid, "0123456789"/"", ({"" })*10) == "")) -  gid = (int)gid; -  -  if(!stringp(uid)) -  u = getpwuid(uid); -  else -  { -  u = getpwnam(uid); -  if(u) -  uid = u[2]; -  } -  -  if(u && !gid) -  gid = u[3]; -  -  if(!u) -  { -  if (uid && (uid != "root")) -  { -  if (intp(uid) && (uid >= 60000)) -  { -  report_warning(sprintf("Privs: User %d is not in the password database.\n" -  "Assuming nobody.\n", uid)); -  // Nobody. -  gid = gid || uid; // Fake a gid also. -  u = ({ "fake-nobody", "x", uid, gid, "A real nobody", "/", "/sbin/sh" }); -  } else { -  error("Unknown user: "+uid+"\n"); -  } -  } else { -  u = ({ "root", "x", 0, gid, "The super-user", "/", "/sbin/sh" }); -  } -  } -  -  if(LOGP) -  report_notice(LOC_M(1, "Change to %s(%d):%d privs wanted (%s), from %s"), -  (string)u[0], (int)uid, (int)gid, -  (string)reason, -  (string)dbt(backtrace()[-2])); -  -  if (u[2]) { - #if efun(cleargroups) -  if (mixed err = catch { cleargroups(); }) -  master()->handle_error (err); - #endif /* cleargroups */ - #if efun(initgroups) -  if (mixed err = catch { initgroups(u[0], u[3]); }) -  master()->handle_error (err); - #endif -  } -  gid = gid || getgid(); -  int err = (int)setegid(new_gid = gid); -  if (err < 0) { -  report_warning(LOC_M(2, "Privs: WARNING: Failed to set the " -  "effective group id to %d!\n" -  "Check that your password database is correct " -  "for user %s(%d),\n and that your group " -  "database is correct.\n"), -  gid, (string)u[0], (int)uid); -  int gid2 = gid; - #ifdef HPUX_KLUDGE -  if (gid >= 60000) { -  /* HPUX has doesn't like groups higher than 60000, -  * but has assigned nobody to group 60001 (which isn't even -  * in /etc/group!). -  * -  * HPUX's libc also insists on filling numeric fields it doesn't like -  * with the value 60001! -  */ -  report_debug("Privs: WARNING: Assuming nobody-group.\n" -  "Trying some alternatives...\n"); -  // Assume we want the nobody group, and try a couple of alternatives -  foreach(({ 60001, 65534, -2 }), gid2) { -  report_debug("%d... ", gid2); -  if (initgroups(u[0], gid2) >= 0) { -  if ((err = setegid(new_gid = gid2)) >= 0) { -  report_debug("Success!\n"); -  break; -  } -  } -  } -  } - #endif /* HPUX_KLUDGE */ -  if (err < 0) { -  report_debug("Privs: Failed\n"); -  error ("Failed to set EGID to %d\n", gid); -  } -  report_debug("Privs: WARNING: Set egid to %d instead of %d.\n", -  gid2, gid); -  gid = gid2; -  } -  if(getgid()!=gid) setgid(gid||getgid()); -  seteuid(new_uid = uid); -  enable_coredumps(1); - #endif /* HAVE_EFFECTIVE_USER */ -  } -  -  void destroy() -  { -  // No need for Privs if the uid has been changed permanently. -  if(getuid()) return; -  - #ifdef PRIVS_DEBUG -  report_debug(sprintf("Privs->destroy()\n" -  "privs_level: %O\n", -  privs_level)); - #endif /* PRIVS_DEBUG */ -  - #ifdef HAVE_EFFECTIVE_USER -  /* Check that we don't increase the privs level */ -  if (p_level >= privs_level) { -  report_error(sprintf("Change back to uid#%d gid#%d from uid#%d gid#%d\n" -  "in wrong order! Saved level:%d Current level:%d\n" -  "Occurs in:\n%s\n", -  saved_uid, saved_gid, new_uid, new_gid, -  p_level, privs_level, -  describe_backtrace(backtrace()))); -  return(0); -  } -  if (p_level != privs_level-1) { -  report_error(sprintf("Change back to uid#%d gid#%d from uid#%d gid#%d\n" -  "Skips privs level. Saved level:%d Current level:%d\n" -  "Occurs in:\n%s\n", -  saved_uid, saved_gid, new_uid, new_gid, -  p_level, privs_level, -  describe_backtrace(backtrace()))); -  } -  privs_level = p_level; -  -  if(LOGP) { -  if (mixed err = catch { -  array bt = backtrace(); -  if (sizeof(bt) >= 2) { -  report_notice(LOC_M(3,"Change back to uid#%d gid#%d, from %s")+"\n", -  saved_uid, saved_gid, dbt(bt[-2])); -  } else { -  report_notice(LOC_M(4,"Change back to uid#%d gid#%d, " -  "from backend")+"\n", saved_uid, saved_gid); -  } -  }) -  master()->handle_error (err); -  } -  - #ifdef PRIVS_DEBUG -  int uid = geteuid(); -  if (uid != new_uid) { -  report_debug("Privs: UID #%d differs from expected #%d\n" -  "%s\n", -  uid, new_uid, describe_backtrace(backtrace())); -  } -  int gid = getegid(); -  if (gid != new_gid) { -  report_debug("Privs: GID #%d differs from expected #%d\n" -  "%s\n", -  gid, new_gid, describe_backtrace(backtrace())); -  } - #endif /* PRIVS_DEBUG */ -  -  seteuid(0); -  array u = getpwuid(saved_uid); - #if efun(cleargroups) -  if (mixed err = catch { cleargroups(); }) -  master()->handle_error (err); - #endif /* cleargroups */ -  if(u && (sizeof(u) > 3)) { -  if (mixed err = catch { initgroups(u[0], u[3]); }) -  master()->handle_error (err); -  } -  setegid(saved_gid); -  seteuid(saved_uid); -  enable_coredumps(1); - #endif /* HAVE_EFFECTIVE_USER */ -  } - #else /* efun(seteuid) */ -  void create(string reason, int|string|void uid, int|string|void gid){} - #endif /* efun(seteuid) */ - } -  +    /* Used by read_config.pike, since there seems to be problems with    * overloading otherwise.    */   protected Privs PRIVS(string r, int|string|void u, int|string|void g)   {    return Privs(r, u, g);   }      // Current Configuration.   Thread.Local current_configuration = Thread.Local();
Roxen.git/server/base_server/roxen.pike:501:    master()->handle_error (err);   #endif    roxenloader.real_exit( exit_code ); // Now we die...   }      private int shutdown_recurse;      // Shutdown Roxen   // exit_code = 0 True shutdown   // exit_code = -1 Restart - private void low_shutdown(int exit_code) + private void low_shutdown(int exit_code, int|void apply_patches)   {    if(shutdown_recurse >= 4)    {    if (mixed err =    catch (report_notice("Exiting roxen (spurious signals received).\n")) ||    catch (stop_all_configurations()))    master()->handle_error (err);    // Zap some of the remaining caches.    destruct(argcache);    destruct(cache); -  +  stop_scan_certs(); +  stop_hourly_maintenance();   #ifdef THREADS -  + #if constant(Filesystem.Monitor.basic) +  stop_fsgarb(); + #endif    if (mixed err = catch (stop_handler_threads()))    master()->handle_error (err);   #endif /* THREADS */    roxenloader.real_exit(exit_code);    }    if (shutdown_recurse++) return;      #ifndef NO_SLOW_REQ_BT    // Turn off the backend thread monitor while we're shutting down.    slow_be_timeout_changed();   #endif    -  +  if ((apply_patches || query("patch_on_restart")) > 0) { +  mixed err = catch { +  foreach(plib->file_list_imported(), mapping(string:mixed) item) { +  report_notice("Applying patch %s...\n", item->metadata->id); +  mixed err = catch { +  plib->install_patch(item->metadata->id, +  "Internal Administrator"); +  }; +  if (err) { +  report_error("Failed to install patch %s: %s\n", +  item->metadata->id, +  describe_backtrace(err)); +  } +  } +  }; +  if (err) { +  master()->handle_error(err); +  } +  } +     if (mixed err = catch(stop_all_configurations()))    master()->handle_error (err);      #ifdef SNMP_AGENT    if(objectp(snmpagent)) {    snmpagent->stop_trap();    snmpagent->disable();    }   #endif       call_out(really_low_shutdown, 0.1, exit_code);   }      private int shutdown_started;      // Perhaps somewhat misnamed, really... This function will close all   // listen ports and then quit. The 'start' script should then start a   // new copy of roxen automatically. - void restart(float|void i, void|int exit_code) + void restart(float|void i, void|int exit_code, void|int apply_patches)   //! Restart roxen, if the start script is running   {    shutdown_started = 1; -  call_out(low_shutdown, i, exit_code || -1); +  call_out(low_shutdown, i, exit_code || -1, apply_patches);   }    - void shutdown(float|void i) + void shutdown(float|void i, void|int apply_patches)   //! Shut down roxen   {    shutdown_started = 1; -  call_out(low_shutdown, i, 0); +  call_out(low_shutdown, i, 0, apply_patches);   }      void exit_when_done()   {    shutdown_started = 1;    report_notice("Interrupt request received.\n"); -  low_shutdown(-1); +  low_shutdown(-1, -1);   }      int is_shutting_down()   //! Returns true if Roxen is shutting down.   {    return shutdown_started;   }         /*    * handle() stuff    */      #ifdef THREADS   // function handle = threaded_handle;    - Thread do_thread_create(string id, function f, mixed ... args) + Thread.Thread do_thread_create(string id, function f, mixed ... args)   {    Thread.Thread t = thread_create(f, @args);    name_thread( t, id );    return t;   }      #if 1   constant Queue = Thread.Queue;   #else   // Shamelessly uses facts about pikes preemting algorithm.
Roxen.git/server/base_server/roxen.pike:630: Inside #if defined(THREADS)
   // above. (Must have an extra ref to the mutex since the    // MutexKey doesn't keep one.)    Thread.Mutex m = Thread.Mutex();    r_cond::wait (m->lock());    }    mixed tmp = buffer[r_ptr];    buffer[r_ptr++] = 0; // Throw away any references.    return tmp;    }    -  mixed tryread() +  mixed try_read()    {    if (!(w_ptr - r_ptr)) return ([])[0];    mixed tmp = buffer[r_ptr];    buffer[r_ptr++] = 0; // Throw away any references.    return tmp;    }       // Warning: This function isn't thread safe.    void write(mixed v)    {
Roxen.git/server/base_server/roxen.pike:664: Inside #if undefined(NO_SLOW_REQ_BT)
  // This is a system to dump all threads whenever a request takes   // longer than a configurable timeout.      protected Pike.Backend slow_req_monitor; // Set iff slow req bt is enabled.   protected float slow_req_timeout, slow_be_timeout;      protected void slow_req_monitor_thread (Pike.Backend my_monitor)   {    // my_monitor is just a safeguard to ensure we don't get multiple    // monitor threads. +  name_thread(this_thread(), "Slow request monitor");    while (slow_req_monitor == my_monitor) -  slow_req_monitor (3600); +  slow_req_monitor (3600.0); +  name_thread(this_thread(), 0);   }      protected mixed slow_be_call_out;      protected void slow_be_before_cb()   {   #ifdef DEBUG    if (this_thread() != backend_thread) error ("Run from wrong thread.\n");   #endif    if (Pike.Backend monitor = slow_be_call_out && slow_req_monitor) {    monitor->remove_call_out (slow_be_call_out);    slow_be_call_out = 0; -  +  +  float backend_rtime = +  (gethrtime() - thread_task_start_times[this_thread()]) / 1E6; +  thread_task_start_times[this_thread()] = 0; +  +  if (backend_rtime > slow_be_timeout) { +  report_slow_thread_finished (this_thread(), backend_rtime);    }    } -  + }      protected void slow_be_after_cb()   {    // FIXME: This should try to compensate for delays due to the pike    // gc, because it just causes noise here.   #ifdef DEBUG    if (this_thread() != backend_thread) error ("Run from wrong thread.\n");   #endif -  if (slow_be_timeout > 0.0) -  if (Pike.Backend monitor = slow_req_monitor) +  if (slow_be_timeout > 0.0) { +  if (Pike.Backend monitor = slow_req_monitor) { +  thread_task_start_times[this_thread()] = gethrtime();    slow_be_call_out = monitor->call_out (dump_slow_req, slow_be_timeout,    this_thread(), slow_be_timeout);    } -  +  } + }      void slow_req_count_changed()   {    Pike.Backend monitor = slow_req_monitor;    int count = query ("slow_req_bt_count");       if (count && monitor) {    // Just a change of the count - nothing to do.    }   
Roxen.git/server/base_server/roxen.pike:759: Inside #if undefined(NO_SLOW_REQ_BT)
   else {    Pike.DefaultBackend->before_callback = 0;    Pike.DefaultBackend->after_callback = 0;    if (Pike.Backend monitor = slow_be_call_out && slow_req_monitor) {    monitor->remove_call_out (slow_be_call_out);    slow_be_call_out = 0;    }    }   }    + protected int last_dump_hrtime; +    protected void dump_slow_req (Thread.Thread thread, float timeout)   {    object threads_disabled = _disable_threads();       int count = query ("slow_req_bt_count");    if (count > 0) set ("slow_req_bt_count", count - 1);       if (thread == backend_thread && !slow_be_call_out) {    // Avoid false alarms for the backend thread if we got here due to    // a race. Should perhaps have something like this for the handler    // threads too, but otoh races are more rare there due to the    // longer timeouts.    }       else { -  report_debug ("###### %s 0x%x has been busy for more than %g seconds.\n", +  string th_name = +  ((thread != backend_thread) && thread_name(thread, 1)) || ""; +  if (sizeof(th_name)) +  th_name = " - " + th_name + " -"; +  report_debug ("###### %s 0x%x%s has been busy for more than %g seconds.\n",    thread == backend_thread ? "Backend thread" : "Thread", -  thread->id_number(), timeout); -  describe_all_threads (0, threads_disabled); +  thread->id_number(), th_name, timeout); +  int hrnow = gethrtime(); +  if ((hrnow - last_dump_hrtime) / 1E6 < slow_req_timeout / 2) { +  describe_thread (thread); +  } else { +  last_dump_hrtime = hrnow; +  mixed err = catch { +  describe_all_threads(0, 1); +  }; +  if (err) master()->handle_error(err);    } -  +  }       threads_disabled = 0; // Paranoia.   }    -  + protected void report_slow_thread_finished (Thread.Thread thread, +  float time_spent) + { +  if (query ("slow_req_bt_count") == 0) { +  return; +  } +  +  string th_name = +  ((thread != backend_thread) && thread_name(thread, 1)) || ""; +  if (sizeof(th_name)) +  th_name = " - " + th_name + " -"; +  +  report_debug ("###### %s 0x%x%s finished after %.2f seconds.\n", +  (thread == backend_thread ? +  "Backend thread" : "Thread"), +  thread->id_number(), th_name, time_spent); + } +    #endif // !NO_SLOW_REQ_BT      // // This is easier than when there are no threads.   // // See the discussion below. :-)   //   // // But there is extra functionality below we really want, though,   // // so let's use that one instead...   // function async_sig_start( function f, int really )   // {   // return lambda( mixed ... args ) {
Roxen.git/server/base_server/roxen.pike:830:   {    return ((functionp (task[0]) ?    sprintf ("%s: %s", Function.defined (task[0]),    master()->describe_function (task[0])) :    programp (task[0]) ?    sprintf ("%s: %s", Program.defined (task[0]),    master()->describe_program (task[0])) :    sprintf ("%O", task[0])) +    "(" +    map (task[1], lambda (mixed arg) -  {return sprintf ("%O", arg);}) * ", " + +  {return RXML.utils.format_short (arg, 200);}) * ", " +    ")");   }      protected mapping(Thread.Thread:int) thread_task_start_times = ([]);      mapping(Thread.Thread:int) get_thread_task_start_times()   {    // Also needed in Admin interface's thread wizard    return thread_task_start_times + ([ ]);   }
Roxen.git/server/base_server/roxen.pike:882:    if(arrayp(h=handle_queue->read()) && h[0]) {    THREAD_WERR(sprintf("Handle thread [%O] calling %s",    id, debug_format_queue_task (h)));    set_locale();    busy_threads++;    thread_flagged_as_busy = 1;    handler_num_runs++;       int start_hrtime = gethrtime();    thread_task_start_times[this_thread()] = start_hrtime; +     float handler_vtime = gauge {   #ifndef NO_SLOW_REQ_BT    if (h[0] != bg_process_queue &&    // Leave out bg_process_queue. It makes a timeout on    // every individual job instead.    (monitor = slow_req_monitor) && slow_req_timeout > 0.0) {    call_out = monitor->call_out (dump_slow_req, slow_req_timeout,    this_thread(), slow_req_timeout);    h[0](@h[1]);    monitor->remove_call_out (call_out);    }    else   #endif    {    h[0](@h[1]);    }    }; -  float handler_rtime = (gethrtime() - start_hrtime)/1E6; +  int end_hrtime = gethrtime(); +  +  float handler_rtime = (end_hrtime - start_hrtime)/1E6;    thread_task_start_times[this_thread()] = 0;    -  + #ifndef NO_SLOW_REQ_BT +  if (slow_req_timeout > 0.0 && +  handler_rtime > slow_req_timeout) { +  report_slow_thread_finished (this_thread(), handler_rtime); +  } + #endif +     h=0;    busy_threads--;    thread_flagged_as_busy = 0;    if (handler_rtime > 0.01) handler_num_runs_001s++;    if (handler_rtime > 0.05) handler_num_runs_005s++;    if (handler_rtime > 0.15) handler_num_runs_015s++;    if (handler_rtime > 0.50) handler_num_runs_05s++;    if (handler_rtime > 1.00) handler_num_runs_1s++;    if (handler_rtime > 5.00) handler_num_runs_5s++;    if (handler_rtime > 15.00) handler_num_runs_15s++;
Roxen.git/server/base_server/roxen.pike:1055:      void release_handler_threads (int numthreads)   //! Releases any handler threads put on hold. If necessary new threads   //! are started to ensure that at least @[numthreads] threads are   //! responding. Threads that haven't arrived to the hold state since   //! @[hold_handler_threads] are considered nonresponding.   {    if (Thread.Condition cond = hold_wakeup_cond) {    // Flush out any remaining hold messages from the queue.    for (int i = handle_queue->size(); i && num_hold_messages; i--) { -  mixed task = handle_queue->tryread(); +  mixed task = handle_queue->try_read();    if (task == 1) num_hold_messages--;    else handle_queue->write (task);    }   #ifdef DEBUG    if (num_hold_messages)    error ("num_hold_messages is bogus (%d).\n", num_hold_messages);   #endif    num_hold_messages = 0;       int blocked_threads = number_of_threads - threads_on_hold;
Roxen.git/server/base_server/roxen.pike:1515:    function func;    array args;    int stopping = 0;       protected void repeat (function func, array args)    {    // Got a minimum of four refs to this:    // o One in the task array in bg_process_queue.    // o One on the stack in the call in bg_process_queue.    // o One as current_object in the stack frame. -  // o One on the stack as argument to _refs. -  int self_refs = _refs (this); +  // o One on the stack as argument to Debug.refs. +  int self_refs = Debug.refs (this);   #ifdef DEBUG    if (self_refs < 4)    error ("Minimum ref calculation wrong - have only %d refs.\n", self_refs);   #endif    if (stopping || (self_refs <= 4) || !func) {    stopping = 2; // Stopped.    return;    }    mixed err = catch {    func (@args);
Roxen.git/server/base_server/roxen.pike:1586:       string _sprintf()    {    return sprintf("BackgroundProcess(%O, %O)", period, func);    }   }         mapping get_port_options( string key )   //! Get the options for the key 'key'. - //! The intepretation of the options is protocol specific. + //! The interpretation of the options is protocol specific.   {    return (query( "port_options" )[ key ] || ([]));   }      void set_port_options( string key, mapping value )   //! Set the options for the key 'key'. - //! The intepretation of the options is protocol specific. + //! The interpretation of the options is protocol specific.   {    mapping q = query("port_options");    q[ key ] = value;    set( "port_options" , q );    save( );   }      #ifdef DEBUG_URL2CONF   #define URL2CONF_MSG(X...) report_debug (X)   #else
Roxen.git/server/base_server/roxen.pike:1748:    raw_url = path;    method = "GET";    raw = "GET " + raw_url + " HTTP/1.1\r\n\r\n";    [port_obj, mapping(string:mixed) url_data] = find_port_for_url (uri, 0);    if (url_data) {    conf = url_data->conf;    if (!conf->inited) conf->enable_all_modules();    if (string config_path = url_data->path)    adjust_for_config_path (config_path);    } +  +  // Update the cached URL base to keep url_base() happy. +  uri->path = (misc->site_prefix_path || "") + "/"; +  uri->query = UNDEFINED; +  uri->fragment = UNDEFINED; +  cached_url_base = sprintf("%s", uri);    return set_path( raw_url );    }       protected string _sprintf()    {    return sprintf("InternalRequestID(conf=%O; not_query=%O)", conf, not_query );    }       protected void create()    {    client = ({ "Roxen" });    prot = "INTERNAL"; -  +  port_obj = InternalProtocol();    method = "GET";    real_variables = ([]);    variables = FakedVariables( real_variables );    root_id = this_object(); -  +  cached_url_base = "internal://0.0.0.0:0/";       misc = ([ "pref_languages": PrefLanguages(),    "cacheable": INITIAL_CACHEABLE,    ]);    connection_misc = ([]);    cookies = CookieJar();    throttle = ([]);    client_var = ([]);    request_headers = ([]);    prestate = (<>);
Roxen.git/server/base_server/roxen.pike:2072:    {    mapping(string:mixed) url_data = find_url_data_for_url (url, 0, id);       if (!url_data) {    // Pick the first default server available. FIXME: This makes    // it impossible to handle the server path correctly.    foreach (configurations, Configuration c)    if (c->query ("default_server")) {    URL2CONF_MSG ("%O %O any default server: %O\n", this, url, c);    if (id) id->misc->defaulted_conf = 3; -  if(!c->inited) -  // FIXME: We can be called from the backend thread, so -  // this should be queued for a handler thread. -  c->enable_all_modules(); +     return c;    }       // if we end up here, there is no default port at all available    // so grab the first configuration that is available at all.    // We choose the last entry in sorted_urls since that's the most    // generic one and therefore probably the best option for a    // fallback.    url_data = urls[sorted_urls[-1]];    if (id) {
Roxen.git/server/base_server/roxen.pike:2100:    url_data->conf);    }       // It's assumed nothing below uses data in this object, since    // find_url_data_for_url might have switched Protocol object.       string config_path = url_data->path;    if (config_path && id && id->adjust_for_config_path)    id->adjust_for_config_path (config_path);    Configuration c = url_data->conf; -  if(!c->inited) -  // FIXME: We can be called from the backend thread, so this -  // should be queued for a handler thread. -  c->enable_all_modules(); +     return c;    }       mixed query_option( string x )    //! Query the port-option 'x' for this port.    {    return query( x );    }       string get_key()
Roxen.git/server/base_server/roxen.pike:2274:    setup (pn, i);    bind (ignore_eaddrinuse);    }       protected string _sprintf( )    {    return "Protocol(" + get_url() + ")";    }   }    - #if constant(SSL.sslfile) - class SSLProtocol - //! Base protocol for SSL ports. Exactly like Port, but uses SSL. + class InternalProtocol + //! Protocol for internal requests that are not linked to any real request.   {    inherit Protocol;    -  +  constant name = "internal"; +  +  constant prot_name = "internal"; +  +  constant supports_ipless = 1; +  constant default_port = 0; +  +  protected void create() +  { +  path = ""; +  port = default_port; +  ip = "0.0.0.0"; +  } + } +  + #if constant(SSL.File) +  + // Some convenience functions. + #if constant(SSL.Constants.fmt_cipher_suites) + constant fmt_cipher_suite = SSL.Constants.fmt_cipher_suite; + constant fmt_cipher_suites = SSL.Constants.fmt_cipher_suites; + #else + protected mapping(int:string) suite_to_symbol = ([]); +  + string fmt_cipher_suite(int suite) + { +  if (!sizeof(suite_to_symbol)) { +  foreach(indices(SSL.Constants), string id) { +  if (has_prefix(id, "SSL_") || has_prefix(id, "TLS_") || +  has_prefix(id, "SSL2_")) { +  suite_to_symbol[SSL.Constants[id]] = id; +  } +  } +  } +  string res = suite_to_symbol[suite]; +  if (res) return res; +  return suite_to_symbol[suite] = sprintf("unknown(%d)", suite); + } +  + string fmt_cipher_suites(array(int) s) + { +  String.Buffer b = String.Buffer(); +  foreach(s, int c) { +  b->add(sprintf(" %-6d: %s\n", c, fmt_cipher_suite(c))); +  } +  return (string)b; + } + #endif +  + class SSLContext { + #if constant(SSL.Context) +  inherit SSL.Context; +  + #if defined(DEBUG) || defined(SSL3_DEBUG) +  SSL.Alert alert_factory(SSL.Connection con, int level, int description, +  SSL.Constants.ProtocolVersion version, +  string|void debug_message) +  { +  if (description != SSL.Constants.ALERT_close_notify) { +  if (debug_message) { +  werror("SSL %s: %s: %s", +  (level == SSL.Constants.ALERT_warning)? +  "WARNING":"ERROR", +  SSL.Constants.fmt_constant(description, "ALERT"), +  debug_message); +  } else { +  werror("SSL %s: %s\n", +  (level == SSL.Constants.ALERT_warning)? +  "WARNING":"ERROR", +  SSL.Constants.fmt_constant(description, "ALERT")); +  } +  } +  return ::alert_factory(con, level, description, version, debug_message); +  } + #endif /* DEBUG || SSL3_DEBUG */ +  + #else +  inherit SSL.context; + #endif + } +  + //! Base protocol for protocols that support upgrading to TLS. + //! + //! Exactly like Port, but contains settings for TLS. + class StartTLSProtocol + { +  inherit Protocol; +     // SSL context -  SSL.context ctx = SSL.context(); +  SSLContext ctx = SSLContext();       int cert_failure;       protected void cert_err_unbind()    {    if (bound > 0) {    port_obj->close();    report_warning ("TLS port %s closed.\n", get_url());    bound = 0;    }
Roxen.git/server/base_server/roxen.pike:2313:    string msg = (MSG); \    array args = ({ARGS}); \    if (sizeof (args)) msg = sprintf (msg, @args); \    report_error ("TLS port %s: %s", get_url(), msg); \    (VAR)->add_warning (msg); \    cert_err_unbind(); \    cert_failure = 1; \    return; \    } while (0)    -  protected void filter_preferred_suites() { + #if constant(SSL.Constants.PROTOCOL_TLS_MAX) +  protected void set_version() +  { +  ctx->min_version = query("ssl_min_version"); +  } + #endif +  +  protected void filter_preferred_suites() +  { + #if constant(SSL.ServerConnection) +  int mode = query("ssl_suite_filter"); +  int bits = query("ssl_key_bits"); +  +  array(int) suites = ({}); +  +  if ((mode & 8) && !ctx->configure_suite_b) { +  // FIXME: Warn: Suite B suites not available. +  mode &= ~8; +  } +  +  if ((mode & 8) && ctx->configure_suite_b) { +  // Suite B. +  switch(mode) { +  case 15: +  // Strict mode. +  ctx->configure_suite_b(bits, 2); +  break; +  case 14: +  // Transitional mode. +  ctx->configure_suite_b(bits, 1); +  break; +  default: +  ctx->configure_suite_b(bits); +  break; +  } +  suites = ctx->preferred_suites; +  +  if (ctx->min_version < query("ssl_min_version")) { +  set_version(); +  } +  } else { +  suites = ctx->get_suites(bits, 1); +  +  // Make sure the min version is restored in case we've +  // switched from Suite B. +  set_version(); +  } +  if (mode & 4) { +  // Ephemeral suites only. +  suites = filter(suites, +  lambda(int suite) { +  return (< +  SSL.Constants.KE_dhe_dss, +  SSL.Constants.KE_dhe_rsa, +  SSL.Constants.KE_ecdhe_ecdsa, +  SSL.Constants.KE_ecdhe_rsa, +  >)[(SSL.Constants.CIPHER_SUITES[suite]||({ -1 }))[0]]; +  }); +  } +  ctx->preferred_suites = suites; + #elif constant(SSL.Constants.CIPHER_aead) +  int bits = query("ssl_key_bits"); +  // NB: The arguments to get_suites() in Pike 7.8 currently differs +  // from the ones in Pike 8.0. +  ctx->preferred_suites = ctx->get_suites(SSL.Constants.SIGNATURE_rsa, bits); + #else   #ifndef ALLOW_WEAK_SSL    // Filter weak and really weak cipher suites.    ctx->preferred_suites -= ({    SSL.Constants.SSL_rsa_with_des_cbc_sha,    SSL.Constants.SSL_dhe_dss_with_des_cbc_sha,    SSL.Constants.SSL_rsa_export_with_rc4_40_md5,    SSL.Constants.TLS_rsa_with_null_sha256,    SSL.Constants.SSL_rsa_with_null_sha,    SSL.Constants.SSL_rsa_with_null_md5,    SSL.Constants.SSL_dhe_dss_export_with_des40_cbc_sha,    SSL.Constants.SSL_null_with_null_null,    });   #endif -  + #endif /* SSL.ServerConnection */ + #ifdef ROXEN_SSL_DEBUG +  report_debug("SSL: Cipher suites enabled for %O:\n" +  "%s\n", +  this_object(), +  fmt_cipher_suites(ctx->preferred_suites)); + #endif    }       void certificates_changed(Variable.Variable|void ignored,    void|int ignore_eaddrinuse)    {    int old_cert_failure = cert_failure; -  +  cert_failure = 0;    -  string raw_keydata; -  array(string) certificates = ({}); -  array(object) decoded_certs = ({}); +  Variable.Variable Keys = getvar("ssl_keys"); +  +  array(int) keypairs = Keys->query(); +  if (!sizeof(keypairs)) { +  // No new-style certificates configured. +  +  // Check if there are old-style certificates; in case of which +  // this is probably an upgrade.    Variable.Variable Certificates = getvar("ssl_cert_file"); -  +  Variable.Variable KeyFile = getvar("ssl_key_file");    -  object privs = Privs("Reading cert file"); +  keypairs = +  CertDB.register_pem_files(Certificates->query() + ({ KeyFile->query() }), +  query("ssl_password"));    -  foreach(map(Certificates->query(), String.trim_whites), string cert_file) { -  string raw_cert; -  SSL3_WERR (sprintf ("Reading cert file %O", cert_file)); -  if( catch{ raw_cert = lopen(cert_file, "r")->read(); } ) -  { -  CERT_WARNING (Certificates, -  LOC_M(8, "Reading certificate file %O failed: %s\n"), -  cert_file, strerror (errno())); -  continue; +  if (!sizeof(keypairs)) { +  // No Old-style certificate configuration found. +  // Fall back to using all known certs. +  keypairs = Keys->get_choice_list();    }    -  object msg = Tools.PEM.pem_msg()->init( raw_cert ); -  object part = msg->parts["CERTIFICATE"] || -  msg->parts["X509 CERTIFICATE"]; -  string cert; +  if (sizeof(keypairs)) { +  // Certificates found. +  Keys->set(keypairs);    -  if (msg->parts["RSA PRIVATE KEY"] || -  msg->parts["DSA PRIVATE KEY"]) { -  raw_keydata = raw_cert; +  save(); +  } else { +  // No certs known to the server. +  // Not reached except in very special circumstances. +  // FIXME: Use anonymous suites? +  report_error ("TLS port %s: %s", get_url(), +  LOC_M(63,"No certificates found.\n")); +  cert_err_unbind(); +  cert_failure = 1; +  return;    } -  -  if (!part || !(cert = part->decoded_body())) -  { -  CERT_WARNING (Certificates, -  LOC_M(10, "No certificate found in %O.\n"), -  cert_file); -  continue; +     } -  certificates += ({ cert }); +     -  // FIXME: Support PKCS7 -  object tbs = Tools.X509.decode_certificate (cert); -  if (!tbs) { -  CERT_WARNING (Certificates, -  LOC_M(13, "Certificate not valid (DER).\n")); -  continue; +  // FIXME: Only do this if there are certs loaded? +  // We must reset the set of certificates. +  // NB: Race condition here where the new SSLContext is +  // live before it has been configured completely. +  ctx = SSLContext(); +  set_version(); +  filter_preferred_suites(); +  +  foreach(keypairs, int keypair_id) { +  array(Crypto.Sign.State|array(string)) keypair = +  CertDB.get_keypair(keypair_id); +  if (!keypair) continue; +  +  [Crypto.Sign.State private_key, array(string) certs] = keypair; +  ctx->add_cert(private_key, certs, ({ name, "*" }));    } -  decoded_certs += ({tbs}); -  } +     -  if (!sizeof(decoded_certs)) { + #if 0 +  // FIXME: How do this in current Pike 8.0? +  if (!sizeof(ctx->cert_pairs)) { +  CERT_ERROR(Certificates, +  LOC_M(71,"No matching keys and certificates found.\n"));    report_error ("TLS port %s: %s", get_url(), -  LOC_M(63,"No certificates found.\n")); +  LOC_M(71,"No matching keys and certificates found.\n"));    cert_err_unbind();    cert_failure = 1;    return;    } -  + #endif    -  Variable.Variable KeyFile = getvar("ssl_key_file"); -  -  if( strlen(KeyFile->query())) { -  SSL3_WERR (sprintf ("Reading key file %O", KeyFile->query())); -  if (catch{ raw_keydata = lopen(KeyFile->query(), "r")->read(); } ) -  CERT_ERROR (KeyFile, -  LOC_M(9, "Reading key file %O failed: %s\n"), -  KeyFile->query(), strerror (errno())); +  if (!bound) { +  bind (ignore_eaddrinuse); +  if (old_cert_failure && bound) +  report_notice (LOC_M(64, "TLS port %s opened.\n"), get_url()); +  if (!bound) +  report_notice("Failed to bind port %s.\n", get_url());    } -  else -  KeyFile = Certificates; +  }    -  privs = 0; +  class CertificateKeyChoiceVariable +  { +  inherit Variable.IntChoice;    -  if (!raw_keydata) -  CERT_ERROR (KeyFile, LOC_M (17,"No private key found.\n")); +  mapping(int:string) get_translation_table() +  { +  array(mapping(string:int|string)) keypairs = CertDB.list_keypairs(); +  return mkmapping(keypairs->id, keypairs->name); +  }    -  object msg = Tools.PEM.pem_msg()->init( raw_keydata ); +  array(int) get_choice_list() +  { +  return CertDB.list_keypairs()->id; +  }    -  SSL3_WERR(sprintf("key file contains: %O", indices(msg->parts))); +  array(string|mixed) verify_set(array(int) new_value) +  { +  if (!sizeof(new_value)) { +  // The list of certificates should never be empty. +  return ({ "Selection reset to all selected.", get_choice_list() }); +  } +  return ::verify_set(new_value); +  }    -  object part; -  if (part = msg->parts["RSA PRIVATE KEY"]) +  protected mapping(Standards.ASN1.Types.Identifier:string) +  parse_dn(Standards.ASN1.Types.Sequence dn)    { -  string key; +  mapping(Standards.ASN1.Types.Identifier:string) ids = ([]); +  foreach(dn->elements, Standards.ASN1.Types.Compound pair) +  { +  if(pair->type_name!="SET" || !sizeof(pair)) continue; +  pair = pair[0]; +  if(pair->type_name!="SEQUENCE" || sizeof(pair)!=2) +  continue; +  if(pair[0]->type_name=="OBJECT IDENTIFIER" && +  pair[1]->value && !ids[pair[0]]) +  ids[pair[0]] = pair[1]->value; +  } +  return ids; +  }    -  if (!(key = part->decoded_body())) -  CERT_ERROR (KeyFile, -  LOC_M(11,"Private rsa key not valid")+" (PEM).\n"); +  protected array(string) render_element(int keypair_id) +  { +  array(Crypto.Sign.State|array(string)) keypair = +  CertDB.get_keypair(keypair_id); +  if (!keypair) { +  return ({ "<td colspan='2'>" + +  LOC_C(1129, "Lost certificate") + +  "</td>" }); +  } +  [Crypto.Sign.State private_key, array(string) certs] = keypair;    -  object rsa = Standards.PKCS.RSA.parse_private_key(key); -  if (!rsa) -  CERT_ERROR (KeyFile, -  LOC_M(11,"Private rsa key not valid")+" (DER).\n"); +  Standards.X509.TBSCertificate tbs = +  Standards.X509.decode_certificate(certs[0]);    -  ctx->rsa = rsa; +  array(string) res = ({});    -  SSL3_WERR(sprintf("RSA key size: %d bits", rsa->rsa_size())); +  if (!tbs) { +  res += ({ "<td colspan='2'><b>" + +  LOC_C(1130, "Invalid certificate") + +  ".</b>" }); +  } else { +  mapping(Standards.ASN1.Types.Identifier:string) dn = +  parse_dn(tbs->subject);    -  if (rsa->rsa_size() > 512) -  { -  /* Too large for export */ -  ctx->short_rsa = Crypto.RSA()->generate_key(512, ctx->random); -  -  // ctx->long_rsa = Crypto.RSA()->generate_key(rsa->rsa_size(), ctx->random); +  string tmp; +  if ((tmp = dn[Standards.PKCS.Identifiers.at_ids.commonName])) { +  res += ({ +  sprintf("<td style='white-space:nowrap'>%s</td>" +  "<td><b><tt>%s</tt></b>", +  LOC_C(1131, "Common Name"), +  Roxen.html_encode_string(tmp)), +  }); +  } else { +  res += ({ "<td colspan='2'>" });    } -  ctx->rsa_mode(); -  filter_preferred_suites(); +     -  array(int) key_matches = -  map(decoded_certs, -  lambda (object tbs) { -  return tbs->public_key->rsa->public_key_equal (rsa); +  res[-1] += sprintf(" (%s, " + LOC_C(1132, "%d bits") + ")</td>", +  Roxen.html_encode_string(private_key->name()), +  private_key->key_size()); +  +  if (tmp = dn[Standards.PKCS.Identifiers.at_ids.organizationName]) { +  if (dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]) { +  tmp += "/" + +  dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]; +  } +  res += ({ +  sprintf("<td style='white-space:nowrap'>%s</td><td>%s</td>", +  LOC_C(1133, "Issued To"), +  Roxen.html_encode_string(tmp)),    }); -  +  } else if (tmp = dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]) { +  res += ({ +  sprintf("<td style='white-space:nowrap'>%s</td><td>%s</td>", +  LOC_C(1133, "Issued To"), +  Roxen.html_encode_string(tmp)), +  }); +  }    -  int num_key_matches; -  // DWIM: Make sure the main cert comes first. -  array(string) new_certificates = allocate(sizeof(certificates)); -  int i,j; -  for (i=0; i < sizeof(certificates); i++) { -  if (key_matches[i]) { -  new_certificates[j++] = certificates[i]; -  num_key_matches++; +  if (tbs->issuer->get_der() == tbs->subject->get_der()) { +  res += ({ +  sprintf("<td style='white-space:nowrap'>" + +  LOC_C(1134, "Issued By") + +  "</td><td>%s</td>", +  LOC_C(1135, "Self-signed")) +  }); +  } else { +  dn = parse_dn(tbs->issuer); +  tmp = dn[Standards.PKCS.Identifiers.at_ids.organizationName]; +  if (dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]) { +  tmp = (tmp?(tmp + "/"):"") + +  dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName];    } -  +  string tmp2 = dn[Standards.PKCS.Identifiers.at_ids.commonName]; +  if (tmp2) { +  if (tmp) { +  tmp = tmp2 + " (" + tmp + ")"; +  } else { +  tmp = tmp2;    } -  for (i=0; i < sizeof(certificates); i++) { -  if (!key_matches[i]) { -  new_certificates[j++] = certificates[i]; +     } -  +  if (tmp) { +  res += ({ +  sprintf("<td style='white-space:nowrap;vertical-align:top'>" + +  LOC_C(1134, "Issued By") + +  "</td><td>%s</td>", +  Roxen.html_encode_string(tmp)), +  });    } -  if( !num_key_matches ) -  CERT_ERROR (KeyFile, -  LOC_M(14, "Certificate and private key do not match.\n")); -  ctx->certificates = new_certificates; +     } -  else if (part = msg->parts["DSA PRIVATE KEY"]) -  { -  string key; +     -  if (!(key = part->decoded_body())) -  CERT_ERROR (KeyFile, -  LOC_M(15,"Private dsa key not valid")+" (PEM).\n"); +  tmp = Roxen.html_encode_string(Calendar.Second(tbs->not_after)-> +  format_time()); +  if (tbs->not_after < time(1)) { +  // Already expired. +  res += ({ +  sprintf("<td>%s</td>" +  "<td><font color='&usr.warncolor;'>%s</font>\n" +  "<img src='&usr.err-3;' /></td>", +  LOC_C(1136, "Expired"), +  tmp), +  }); +  } else if (tbs->not_after < time(1) + (3600 * 24 * 30)) { +  // Expires within 30 days. +  res += ({ +  sprintf("<td>%s</td>" +  "<td><font color='&usr.warncolor;'>%s</font>\n" +  "<img src='&usr.err-2;' /></td>", +  LOC_C(1137, "Expires"), +  tmp), +  }); +  } else { +  res += ({ +  sprintf("<td>%s</td><td>%s</td>", LOC_C(1137, "Expires"), tmp), +  }); +  }    -  object dsa = Standards.PKCS.DSA.parse_private_key(key); -  if (!dsa) -  CERT_ERROR (KeyFile, -  LOC_M(15,"Private dsa key not valid")+" (DER).\n"); +  mapping keypair_metadata = CertDB.get_keypair_metadata(keypair_id);    -  SSL3_WERR(sprintf("Using DSA key.")); +  array(string) paths = +  keypair_metadata->certs->pem_path + +  ({ keypair_metadata->key->pem_path }); +  paths = Array.uniq(paths); +  paths = replace(paths, 0, "__LOST__"); +  paths = map(paths, lfile_path); +  res += ({ +  sprintf("<td style='vertical-align:top'>%s</td><td>%s</td>", +  LOC_C(1138, "Path(s)"), +  map(paths, lambda(string p) { +  if (p) +  return "<tt>" + Roxen.html_encode_string(p) + "</tt>"; +  else +  return +  "<font color='&usr.warncolor;'>" + +  LOC_C(1139, "Lost file") + +  "</font>"; +  }) * "<br/>") +  }); +  }    -  //dsa->use_random(ctx->random); -  ctx->dsa = dsa; -  /* Use default DH parameters */ - #if constant(SSL.Cipher) -  ctx->dh_params = SSL.Cipher.DHParameters(); - #else -  ctx->dh_params = SSL.cipher()->dh_parameters(); - #endif +  return res; +  }    -  ctx->dhe_dss_mode(); -  filter_preferred_suites(); -  -  // FIXME: Add cert <-> private key check. -  -  ctx->certificates = certificates; +  string render_form(RequestID id, void|mapping additional_args) { +  array(string) current = map(query(), _name); +  string res = "<table width='100%'>\n"; +  foreach( get_choice_list(); int i; mixed elem ) { +  if (i != 0) { +  res += "<tr><td colspan='3'><hr/></td></tr>\n";    } -  else -  CERT_ERROR (KeyFile, LOC_M(17,"No private key found.\n")); +  mapping m = ([ +  "type": "checkbox", +  "name": path(), +  "value": _name(elem), +  ]); +  if(has_value(current, m->value)) { +  m->checked="checked"; +  current -= ({ m->value }); +  } +  array(string) el_rows = render_element(elem); +  res += sprintf("<tr><td rowspan='%d'>%s</td>" +  "%s" +  "</tr>\n", +  sizeof(el_rows), +  Roxen.make_tag( "input", m), +  el_rows[0]); +  foreach(el_rows[1..], string row) { +  res += sprintf("<tr>%s</tr>", row); +  } +  } +  // Make an entry for the current values if they're not in the list, +  // to ensure that the value doesn't change as a side-effect by +  // another change. +  foreach(current, string value) { +  mapping m = ([ +  "type": "checkbox", +  "name": path(), +  "value": value, +  "checked": "checked", +  ]); +  string title = sprintf(LOC_C(1121,"(stale value %s)"), value); +  res += sprintf("<tr><td>%s</td><td>%s</td></tr>\n", +  Roxen.make_tag( "input", m), +  Roxen.html_encode_string(title)); +  } +  return res + "</table>"; +  }    - #if EXPORT -  ctx->export_mode(); - #endif -  -  if (!bound) { -  bind (ignore_eaddrinuse); -  if (old_cert_failure && bound) -  report_notice (LOC_M(64, "TLS port %s opened.\n"), get_url()); +  protected void create( void|int _flags, void|LocaleString std_name, +  void|LocaleString std_doc ) +  { +  ::create(({}), UNDEFINED, _flags, std_name, std_doc);    }    }    -  + #if 1 +  // Old-style SSL Certificate variables. +  // FIXME: Keep these around for at least a few major versions (10 years?).    class CertificateListVariable    {    inherit Variable.FileList;       string doc()    {    return sprintf(::doc() + "\n",    combine_path(getcwd(), "../local"),    getcwd());    }
Roxen.git/server/base_server/roxen.pike:2532:    {    inherit Variable.String;       string doc()    {    return sprintf(::doc() + "\n",    combine_path(getcwd(), "../local"),    getcwd());    }    } + #endif    -  SSL.sslfile accept() -  { -  Stdio.File q = ::accept(); -  if (q) -  return SSL.sslfile (q, ctx); -  return 0; -  } -  -  protected void bind (void|int ignore_eaddrinuse) -  { -  // Don't bind if we don't have correct certs. -  if (!ctx->certificates) return; -  ::bind (ignore_eaddrinuse); -  } -  +     void create(int pn, string i, void|int ignore_eaddrinuse)    {    ctx->random = Crypto.Random.random_string;    -  filter_preferred_suites(); -  +     set_up_ssl_variables( this_object() );    -  +  // NB: setup() calls restore() which initializes the variables +  // created above.    ::setup(pn, i);    -  + #if constant(SSL.Constants.PROTOCOL_TLS_MAX) +  set_version(); + #endif +  +  filter_preferred_suites(); +     certificates_changed (0, ignore_eaddrinuse);       // Install the change callbacks here to avoid duplicate calls    // above.    // FIXME: Both variables ought to be updated on save before the    // changed callback is called. Currently you can get warnings    // that the files don't match if you update both variables    // at the same time. -  +  getvar ("ssl_keys")->set_changed_callback(certificates_changed);    getvar ("ssl_cert_file")->set_changed_callback (certificates_changed);    getvar ("ssl_key_file")->set_changed_callback (certificates_changed); -  +  + #if constant(SSL.Constants.CIPHER_aead) +  getvar("ssl_key_bits")->set_changed_callback(filter_preferred_suites); + #endif + #if constant(SSL.ServerConnection) +  getvar("ssl_suite_filter")->set_changed_callback(filter_preferred_suites); + #endif + #if constant(SSL.Constants.PROTOCOL_TLS_MAX) +  getvar("ssl_min_version")->set_changed_callback(set_version); + #endif    }       string _sprintf( )    { -  +  return "StartTLSProtocol(" + get_url() + ")"; +  } + } +  + class SSLProtocol + //! Base protocol for SSL ports. + //! + //! Exactly like Port, but uses SSL. + { +  inherit StartTLSProtocol; +  +  SSL.File accept() +  { +  Stdio.File q = ::accept(); +  if (q) { +  SSL.File ssl = SSL.File(q, ctx); +  ssl->accept(); +  return ssl; +  } +  return 0; +  } +  + #if constant(SSL.Connection) +  protected void bind (void|int ignore_eaddrinuse) +  { +  // Don't bind if we don't have correct certs. +  // if (!sizeof(ctx->cert_pairs)) return; +  ::bind (ignore_eaddrinuse); +  } + #else +  protected void bind (void|int ignore_eaddrinuse) +  { +  // Don't bind if we don't have correct certs. +  if (!ctx->certificates) return; +  ::bind (ignore_eaddrinuse); +  } + #endif +  +  string _sprintf( ) +  {    return "SSLProtocol(" + get_url() + ")";    }   }   #endif      mapping(string:program/*(Protocol)*/) build_protocols_mapping()   {    mapping protocols = ([]);    int st = gethrtime();    report_debug("Protocol handlers ... \b");
Roxen.git/server/base_server/roxen.pike:2611: Inside #if undefined(DEBUG)
   mixed `->( string x )    {    if(!real) realize();    return predef::`->(real, x);    }    };   #endif    foreach( glob( "prot_*.pike", get_dir("protocols") ), string s )    {    sscanf( s, "prot_%s.pike", s ); - #if !constant(SSL.sslfile) + #if !constant(SSL.File)    switch( s )    {    case "https":    case "ftps":    continue;    }   #endif    report_debug( "\b%s \b", s );       catch
Roxen.git/server/base_server/roxen.pike:2633: Inside #if defined(DEBUG)
  #ifdef DEBUG    protocols[ s ] = (program)("protocols/prot_"+s+".pike");   #else    protocols[ s ] = lazy_load( ("protocols/prot_"+s+".pike"),s );   #endif    };    }    foreach( glob("prot_*.pike",get_dir("../local/protocols")||({})), string s )    {    sscanf( s, "prot_%s.pike", s ); - #if !constant(SSL.sslfile) + #if !constant(SSL.File)    switch( s )    {    case "https":    case "ftps":    continue;    }   #endif    report_debug( "\b%s \b", s );    catch {   #ifdef DEBUG
Roxen.git/server/base_server/roxen.pike:3212:    if( conf )    {    if( conf->error_log )    conf->error_log[log_index] += ({ log_time });    }       if(errtype >= 1)    report_debug( s );   }    + protected BackgroundProcess hourly_maintenance_process; +  + protected void clean_error_log(mapping(string:array(int)) log, +  mapping(string:int) cutoffs) + { +  if (!log || !sizeof(log)) return; +  foreach(cutoffs; string prefix; int cutoff) { +  foreach(log; string key; array(int) times) { +  if (!has_prefix(key, prefix)) continue; +  int sz = sizeof(times); +  times = filter(times, `>=, cutoff); +  if (sizeof(times) == sz) continue; +  // NB: There's a race here, where newly triggered errors may be lost. +  // It's very unlikely to be a problem in practice though. +  if (!sizeof(times)) { +  m_delete(log, key); +  } else { +  log[key] = times; +  } +  } +  } + } +  + protected void error_log_cleaner() + { +  mapping(string:int) cutoffs = ([ +  "1,": time(1) - 3600*24*7, // Keep notices for 7 days. +  ]); +  +  // First the global error_log. +  clean_error_log(error_log, cutoffs); +  +  // Then all configurations and modules. +  foreach(configurations, Configuration conf) { +  clean_error_log(conf->error_log, cutoffs); +  +  foreach(indices(conf->otomod), RoxenModule mod) { +  clean_error_log(mod->error_log, cutoffs); +  } +  } + } +  + protected void patcher_report_notice(string msg, mixed ... args) + { +  if (sizeof(args)) msg = sprintf(msg, @args); +  report_notice(RoxenPatch.wash_output(msg)); + } +  + protected void patcher_report_error(string msg, mixed ... args) + { +  if (sizeof(args)) msg = sprintf(msg, @args); +  report_error(RoxenPatch.wash_output(msg)); + } +  + RoxenPatch.Patcher plib = +  RoxenPatch.Patcher(patcher_report_notice, patcher_report_error, +  getcwd(), getenv("LOCALDIR")); +  + protected void hourly_maintenance() + { +  error_log_cleaner(); +  +  if (query("auto_fetch_rxps")) { +  plib->import_file_http(); +  } + } +  + protected void start_hourly_maintenance() + { +  if (hourly_maintenance_process) return; +  +  // Start a background process that performs maintenance tasks every hour +  // (eg cleaning the error log). +  hourly_maintenance_process = BackgroundProcess(3600, hourly_maintenance); + } +  + protected void stop_hourly_maintenance() + { +  if (hourly_maintenance_process) { +  hourly_maintenance_process->stop(); +  hourly_maintenance_process = UNDEFINED; +  } + } +    // When was Roxen started?   int boot_time =time();   int start_time =time();      string version()   {   #ifndef NSERIOUS    return query("default_ident")?real_version:query("ident");   #else    multiset choices=(<>);
Roxen.git/server/base_server/roxen.pike:3316: Inside #if undefined(__NT__)
     #ifndef __NT__   protected int abs_started;   protected int handlers_alive;      protected void low_engage_abs()   {    report_debug("**** %s: ABS exiting roxen!\n\n",    ctime(time()) - "\n");    _exit(1); // It might not quit correctly otherwise, if it's -  // locked up +  // locked up. Note that this also inhibits the delay +  // caused by the possible automatic installation of +  // any pending patches.   }      protected void engage_abs(int n)   {    if (!query("abs_engage")) {    abs_started = 0;    report_debug("Anti-Block System Disabled.\n");    return;    }    report_debug("**** %s: ABS engaged!\n"
Roxen.git/server/base_server/roxen.pike:3345: Inside #if undefined(__NT__)
   // Catch for paranoia reasons.    describe_all_threads();    })    master()->handle_error (err);   #ifdef THREADS    report_debug("\nHandler queue:\n");    if (mixed err = catch {    t = alarm(20); // Restart the timeout timer.    array(mixed) queue = handle_queue->buffer[handle_queue->r_ptr..];    foreach(queue, mixed v) { -  if (!v) continue; +  if (!v) { +  // Either an entry past the write pointer, or an entry that +  // has been zapped by a handler thread during our processing. +  continue; +  }    if (!arrayp(v)) {    report_debug(" *** Strange entry: %O ***\n", v);    } else {    report_debug(" %{%O, %}\n", v/({}));    }    }    })    master()->handle_error (err);   #endif -  +  report_debug("\nPending call_outs:\n"); +  if (mixed err = catch { +  t = alarm(20); // Restart the timeout timer. +  foreach(call_out_info(), array info) { +  report_debug(" %4d seconds: %O(%{%O, %})\n", +  info[0], info[2], info[3..]); +  } +  }) +  master()->handle_error(err);    low_engage_abs();   }    -  + //! Called to indicate that the handler threads are alive. + //! + //! Usually called automatically via @[handle()] by @[restart_if_stuck()], + //! but may need to be called by hand when handler threads are intentionally + //! blocked for a longer time (eg via long-lived global mutex locks). + //! + //! Do not call unless you know what you are doing. + void handler_ping() + { +  handlers_alive = time(); + } +    void restart_if_stuck (int force)   //! @note   //! Must be called from the backend thread due to Linux peculiarities.   {    remove_call_out(restart_if_stuck);    if (!(query("abs_engage") || force))    return;    if(!abs_started)    {    abs_started = 1;
Roxen.git/server/base_server/roxen.pike:3383: Inside #if undefined(__NT__)
  // return a[2] == restart_if_stuck; }));    signal(signum("SIGALRM"), engage_abs);    int t = alarm (60*query("abs_timeout")+20);    // werror("alarm: %d seconds left, set to %d\n", t, 60*query("abs_timeout")+20);    if ((time(1) - handlers_alive) > 60*query("abs_timeout")) {    // The handler installed below hasn't run.    report_debug("**** %s: ABS: Handlers are dead!\n",    ctime(time()) - "\n");    engage_abs(0);    } -  handle(lambda() { handlers_alive = time(); }); +  handle(handler_ping);   }   #endif      #if constant(ROXEN_MYSQL_SUPPORTS_UNICODE)   // NOTE: We need to mark binary data as binary in case   // the Mysql character_set_connection has been   // set to anything other than "latin1".   #define MYSQL__BINARY "_binary"   #else   #define MYSQL__BINARY ""
Roxen.git/server/base_server/roxen.pike:3886:    if( alpha )    enc_args->alpha = alpha;       foreach( glob( "*-*", indices(args)), string n )    if(sscanf(n, "%*[^-]-%s", string opt ) == 2)    if( opt != "alpha" )    enc_args[opt] = (int)args[n];       switch(format)    { + #if constant(Image.WebP) && constant(Image.WebP.encode) +  case "webp": +  // Only mixed case module +  data = Image.WebP.encode( reply, enc_args ); +  break; + #endif    case "wbf":    format = "wbmp";    case "wbmp":    Image.Colortable bw=Image.Colortable( ({ ({ 0,0,0 }),    ({ 255,255,255 }) }) );    bw->floyd_steinberg();    data = Image.WBF.encode( bw->map( reply ), enc_args );    break;    case "gif":   #if constant(Image.GIF) && constant(Image.GIF.encode)
Roxen.git/server/base_server/roxen.pike:3981:    {    if(!stringp(data)) return;   #ifdef ARG_CACHE_DEBUG    werror("store %O (%d bytes)\n", id, strlen(data) );   #endif    meta_cache_insert( id, meta );    string meta_data = encode_value( meta );   #ifdef ARG_CACHE_DEBUG    werror("Replacing entry for %O\n", id );   #endif -  QUERY("REPLACE INTO "+name+ +  if (sizeof(data) <= 8*1024*1024) { +  // Should fit in the 16 MB query limit without problem. +  // Albeit it might trigger a slow query entry for large +  // entries. +  QUERY("REPLACE INTO " + name +    " (id,size,atime,meta,data) VALUES"    " (%s,%d,UNIX_TIMESTAMP()," MYSQL__BINARY "%s," MYSQL__BINARY "%s)",    id, strlen(data)+strlen(meta_data), meta_data, data ); -  +  } else { +  // We need to perform multiple queries.   #ifdef ARG_CACHE_DEBUG -  +  werror("Writing %d bytes of padding for %s.\n", sizeof(data), id); + #endif +  array(string) a = data/(8.0*1024*1024); +  // NB: We clear the meta field to ensure that the entry +  // is invalid while we perform the insert. +  QUERY("REPLACE INTO " + name + +  " (id,size,atime,meta,data) VALUES" +  " (%s,%d,UNIX_TIMESTAMP(),'',SPACE(%d))", +  id, strlen(data)+strlen(meta_data), sizeof(data)); +  int pos; +  foreach(a, string frag) { + #ifdef ARG_CACHE_DEBUG +  werror("Writing fragment at position %d for %s.\n", pos, id); + #endif +  QUERY("UPDATE " + name + +  " SET data = INSERT(data, %d, %d, "MYSQL__BINARY "%s)" +  " WHERE id = %s", +  pos+1, sizeof(frag), frag, id); +  pos += sizeof(frag); +  } +  /* Set the meta data field to a valid value to enable the entry. */ + #ifdef ARG_CACHE_DEBUG +  werror("Writing metadata for %s.\n", id); + #endif +  QUERY("UPDATE " + name + +  " SET meta = " MYSQL__BINARY "%s" +  " WHERE id = %s", +  meta_data, id); +  } + #ifdef ARG_CACHE_DEBUG    array(mapping(string:string)) q =    QUERY("SELECT meta, data FROM " + name +    " WHERE id = %s", id);    if (!q || sizeof(q) != 1) {    werror("Unexpected result size: %d\n",    q && sizeof(q));    } else {    if (q[0]->meta != meta_data) {    werror("Meta data differs: %O != %O\n",    meta_data, q[0]->meta);    }    if (q[0]->data != data) { -  werror("Data differs: %O != %O\n", -  data, q[0]->data); +  string d = q[0]->data; +  int i; +  int cnt; +  for (i = 0; i < sizeof(data); i++) { +  if (data[i] == d[i]) continue; +  werror("Data differs at offset %d: %d != %d\n", +  i, data[i], d[i]); +  if (cnt++ > 10) break;    }    } -  +  }   #endif    }       protected mapping restore_meta( string id, RequestID rid )    {    if( array item = meta_cache[ id ] )    {    item[ 1 ] = time(1); // Update cached atime.    return item[ 0 ];    }      #ifdef ARG_CACHE_DEBUG    werror("restore meta %O\n", id );   #endif    array(mapping(string:string)) q = -  QUERY("SELECT meta, UNIX_TIMESTAMP() - atime as atime_diff " +  QUERY("SELECT meta, UNIX_TIMESTAMP()-CAST(atime AS SIGNED) AS atime_diff "    "FROM "+name+" WHERE id=%s", id );       string s;    if(!sizeof(q) || !strlen(s = q[0]->meta))    return 0;       mapping m;    if (catch (m = decode_value (s)))    {    report_error( "Corrupt data in cache-entry "+id+".\n" );
Roxen.git/server/base_server/roxen.pike:4061:    }       void flush(int|void age)    //! Flush the cache. If an age (an integer as returned by    //! @[time()]) is provided, only images with their latest access before    //! that time are flushed.    {    int num;   #if defined(DEBUG) || defined(IMG_CACHE_DEBUG)    int t = gethrtime(); -  report_debug("Cleaning "+name+" image cache ... "); +    #endif    sync_meta();    uid_cache = ([]);    rst_cache = ([]);    if( !age )    { -  +  QUERY( "DELETE FROM "+name );   #if defined(DEBUG) || defined(IMG_CACHE_DEBUG) -  report_debug("cleared\n"); +  int msec = (gethrtime() - t) / 1000; +  report_debug("Image cache %s emptied (%dms).\n", name, msec);   #endif -  QUERY( "DELETE FROM "+name ); +     num = -1;    return;    }       array(string) ids =    QUERY( "SELECT id FROM "+name+" WHERE atime < "+age)->id;       num = sizeof( ids );       int q;
Roxen.git/server/base_server/roxen.pike:4108: Inside #if 0
   // them, really, but it might be nice not to throw an error, at    // least.   #if defined(DEBUG) || defined(IMG_CACHE_DEBUG)    report_debug("Optimizing database ... ", name);   #endif    QUERY( "OPTIMIZE TABLE "+name );    };   #endif      #if defined(DEBUG) || defined(IMG_CACHE_DEBUG) -  report_debug("%s removed (%dms)\n", -  (num==-1?"all":num?(string)num:"none"), -  (gethrtime()-t)/1000); +  int msec = (gethrtime() - t) / 1000; +  if (num || (msec > 500)) { +  report_debug("Image cache %s cleaned: %s removed (%dms)\n", +  name, +  (num == -1 ? "all" : num ? (string) num : "none"), msec); +  }   #endif    }       array(int) status(int|void age)    //! Return the total number of images in the cache, their cumulative    //! sizes in bytes and, if an age time_t was supplied, the number of    //! images that has not been accessed after that time is returned    //! (see <ref>flush()</ref>). (Three integers are returned    //! regardless of whether an age parameter was given.)    {
Roxen.git/server/base_server/roxen.pike:4249: Inside #if defined(ARG_CACHE_DEBUG)
  #ifdef ARG_CACHE_DEBUG    werror("data: %O id: %O\n", na, id );   #endif    if(! (res=restore( na,id )) )    {    mixed err;    if (nodraw || (err = catch {    if (mapping res = draw( na, id ))    return res;    })) { -  // File not found. -  -  if(arrayp(err) && sizeof(err) && stringp(err[0])) -  { + #ifdef ARG_CACHE_DEBUG +  werror("draw() failed with error: %s\n", +  describe_backtrace(err)); + #endif +  if (objectp (err) && err->is_RXML_Backtrace && !RXML_CONTEXT) { +  // If we get an rxml error and there's no rxml context then +  // we're called from a direct request to the image cache. +  // The error ought to have been reported in the page that +  // generated the link to the image cache, but since it's too +  // late for that now, we just log it as a (brief) server +  // error with the referring page. +  string errmsg = "Error in " + name + " image generation: " + +  err->msg; +  if (sizeof (id->referer)) +  errmsg += " Referrer: " + id->referer[0]; +  report_error (errmsg + "\n"); +  return 0; +  } else if (arrayp(err) && sizeof(err) && stringp(err[0])) {    if (sscanf(err[0], "Requesting unknown key %s\n",    string message) == 1)    { -  +  // File not found.    report_debug("Requesting unknown key %s %O from %O\n",    message,    id->not_query,    (sizeof(id->referer)?id->referer[0]:"unknown page"));    return 0;    } -  if (sscanf(err[0], "Failed to load specified image [\"%s\"]\n", -  string message) == 1) -  { -  report_debug("Failed to load specified image %O from %O - referrer %O\n", -  message, -  id->not_query, -  (sizeof(id->referer)?id->referer[0]:"unknown page")); -  return 0; +     } -  + #ifdef ARG_CACHE_DEBUG +  werror("Rethrowing error...\n"); + #endif +  throw (err);    } -  report_debug("Error in draw: %s\n", describe_backtrace(err)); -  return 0; -  } +     if( !(res = restore( na,id )) ) { -  error("Draw callback %O did not generate any data.\n" +  report_error("Draw callback %O did not generate any data.\n"    "na: %O\n"    "id: %O\n",    draw_function, na, id); -  +  return 0;    }    }    res->stat = ({ 0, 0, 0, 900000000, 0, 0, 0, 0, 0 });       // Setting the cacheable flag is done in order to get headers sent which    // cause the image to be cached in the client even when using https    // sessions. -  RAISE_CACHE(INITIAL_CACHEABLE); +  // +  // NB: Raise it above INITIAL_CACHEABLE to force an Expires header. +  RAISE_CACHE(31557600); // A year.       // With the new (5.0 and newer) arg-cache enabled by default we can    // allow authenticated images in the protocol cache. At this point    // http.pike will have cleared it so re-enable explicitly.    PROTO_CACHE();       return res;    }       mapping metadata( array|string|mapping data,
Roxen.git/server/base_server/roxen.pike:4377:    error("Expected mapping as the first element of the argument array\n");    update_args( data[0] );    ci = map( map( data, tomapp ), argcache->store, timeout )*"$";    } else    ci = data;    update_args = 0; // To avoid garbage.       if( zero_type( uid_cache[ ci ] ) )    {    uid_cache[ci] = user; -  if( catch(QUERY("INSERT INTO "+name+" " -  "(id,uid,atime) VALUES (%s,%s,UNIX_TIMESTAMP())", -  ci, user||"")) ) -  QUERY( "UPDATE "+name+" SET uid=%s WHERE id=%s", -  user||"", ci ); +  // Make sure to only update the entry if it does not already +  // exists or has wrong uid. Allways updating the table will +  // casue mysql to lock the table and cause a potential gobal +  // ImageCache stall. +  string uid = user || ""; +  array q = QUERY("SELECT uid from "+name+" where id=%s", ci); +  if(!sizeof(q) || (sizeof(q) && q[0]->uid != uid)) { +  QUERY("INSERT INTO "+name+" " +  "(id,uid,atime) VALUES (%s,%s,UNIX_TIMESTAMP()) " +  "ON DUPLICATE KEY UPDATE uid=%s", +  ci, uid, uid);    } -  +  }      #ifndef NO_ARG_CACHE_SB_REPLICATE    if(id->misc->persistent_cache_crawler || id->misc->do_replicate_argcache) {    // Force an update of rep_time for the requested arg cache id.    foreach(ci/"$", string key) {   #if ARGCACHE_DEBUG    werror("Request for id %O from prefetch crawler.\n", key);   #endif /* ARGCACHE_DEBUG */    argcache->refresh_arg(key);    }
Roxen.git/server/base_server/roxen.pike:4425:       master()->resolv("DBManager.is_module_table")    ( 0,"local",name,"Image cache for "+name);       QUERY("CREATE TABLE "+name+" ("    "id CHAR(64) NOT NULL PRIMARY KEY, "    "size INT UNSIGNED NOT NULL DEFAULT 0, "    "uid CHAR(32) NOT NULL DEFAULT '', "    "atime INT UNSIGNED NOT NULL DEFAULT 0,"    "meta MEDIUMBLOB NOT NULL DEFAULT ''," -  "data MEDIUMBLOB NOT NULL DEFAULT ''," +  "data LONGBLOB NOT NULL DEFAULT '',"    "INDEX atime_id (atime, id)"    ")" );    }    -  +  // Inhibit backups of this table. +  master()->resolv("DBManager.inhibit_backups")("local", name); +     // Create index in old databases. Index is used when flushing old    // entries. Column 'id' is included in index in order to avoid    // reading data file.    array(mapping(string:mixed)) res = QUERY("SHOW INDEX FROM " + name);    if(search(res->Key_name, "atime_id") < 0) {    report_debug("Updating " + name + " image cache: "    "Adding index atime_id on %s... ", name);    int start_time = gethrtime();    QUERY("CREATE INDEX atime_id ON " + name + " (atime, id)");    report_debug("complete. [%f s]\n", (gethrtime() - start_time)/1000000.0);    report_debug("Updating " + name + " image cache: "    "Dropping index atime on %s... ", name);    start_time = gethrtime();    QUERY("DROP INDEX atime ON " + name);    report_debug("complete. [%f s]\n", (gethrtime() - start_time)/1000000.0);    } -  +  res = QUERY("SHOW COLUMNS FROM " + name + " WHERE Field = 'data'"); +  if (lower_case(res[0]->Type) != "longblob") { +  report_debug("Updating " + name + " image cache: " +  "Increasing maximum blob size..."); +  int start_time = gethrtime(); +  QUERY("ALTER TABLE " + name + +  " MODIFY COLUMN data LONGBLOB NOT NULL DEFAULT ''"); +  report_debug("complete. [%f s]\n", (gethrtime() - start_time)/1000000.0);    } -  +  }       Sql.Sql get_db()    {    return dbm_cached_get("local");    }       protected void init_db( )    {    catch(sync_meta());    setup_tables();
Roxen.git/server/base_server/roxen.pike:4468:       void do_cleanup( )    {    // Flushes may be costly in large sites (at least the OPTIMIZE TABLE    // command) so schedule next run sometime after 04:30 the day after    // tomorrow.    //    // Note: The OPTIMIZE TABLE step has been disabled. /mast    int now = time();    mapping info = localtime(now); -  int wait = (int) ((24 - info->hour) + 24 + 4.5) * 3600 + random(500); +  int wait = (int) (((24 - info->hour) + 24 + 4.5) % 24) * 3600 + random(500);    background_run(wait, do_cleanup);       // Remove items older than one week    flush(now - 7 * 3600 * 24);    }       void create( string id, function draw_func )    //! Instantiate an image cache of your own, whose image files will    //! be stored in a table `id' in the cache mysql database,    //!    //! The `draw_func' callback passed will be responsible for    //! (re)generation of the images in the cache. Your draw callback    //! may take any arguments you want, depending on the first argument    //! you give the <ref>store()</ref> method, but its final argument    //! will be the RequestID object. -  +  //! +  //! @note +  //! Use @[RXML.run_error] or @[RXML.parse_error] within the draw +  //! function to throw user level drawing errors, e.g. invalid or +  //! missing images or argument errors. If it's called within a +  //! graphics tag then the error is thrown directly and reported +  //! properly by the rxml evaluator. If it's called later, i.e. in a +  //! direct request to the image cache, then it is catched by the +  //! @[ImageCache] functions and reported in as good way as possible, +  //! i.e. currently briefly in the debug log.    {    name = id;    draw_function = draw_func;    init_db();    // Support that the 'local' database moves.    master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db );       // Always remove entries that are older than one week.    background_run( 10, do_cleanup );    }
Roxen.git/server/base_server/roxen.pike:4563:    "atime DATETIME NOT NULL, "    "rep_time DATETIME NOT NULL, "    "sync_time INT NULL, "    "timeout INT NULL, "    "contents MEDIUMBLOB NOT NULL, "    " INDEX(timeout),"    " INDEX(sync_time)"    ")");    }    +  // Inhibit backups of the arguments2 table. +  master()->resolv("DBManager.inhibit_backups") +  ("local", name + "2"); +     if (catch (QUERY ("SELECT rep_time FROM " + name + "2 LIMIT 0")))    {    // Upgrade a table without rep_time.    QUERY ("ALTER TABLE " + name + "2"    " ADD rep_time DATETIME NOT NULL"    " AFTER atime");    }       if (catch (QUERY ("SELECT timeout FROM " + name + "2 LIMIT 0")))    {
Roxen.git/server/base_server/roxen.pike:4979:       void refresh_arg(string id)    //! Indicate that the entry @[id] needs to be included in the next    //! @[write_dump]. @[id] must be an existing entry.    {    GET_DB();    QUERY("UPDATE "+name+"2 SET rep_time=NOW() WHERE id = %s", id);    }   }    - mapping cached_decoders = ([]); + mapping(string:Charset.Decoder) cached_decoders = ([]);   string decode_charset( string charset, string data )   {    // FIXME: This code is probably not thread-safe!    if( charset == "iso-8859-1" ) return data;    if( !cached_decoders[ charset ] ) -  cached_decoders[ charset ] = Locale.Charset.decoder( charset ); +  cached_decoders[ charset ] = Charset.decoder( charset );    data = cached_decoders[ charset ]->feed( data )->drain();    cached_decoders[ charset ]->clear();    return data;   }      //! Check if a cache key has been marked invalid (aka stale).   int(0..1) invalidp(CacheKey key)   {    catch {    return !key || (key->invalidp && key->invalidp());
Roxen.git/server/base_server/roxen.pike:5081:    dump( "base_server/newdecode.pike" );    dump( "base_server/read_config.pike" );    dump( "base_server/global_variables.pike" );    dump( "base_server/module_support.pike" );    dump( "base_server/socket.pike" );    dump( "base_server/cache.pike" );    dump( "base_server/supports.pike" );    dump( "base_server/hosts.pike");    dump( "base_server/language.pike");    - #ifndef __NT__ -  if(!getuid()) -  add_constant("Privs", Privs); -  else - #endif /* !__NT__ */ -  add_constant("Privs", class { -  void create(string reason, int|string|void uid, int|string|void gid) {} -  }); -  -  +     DDUMP( "base_server/roxenlib.pike");    DDUMP( "etc/modules/Dims.pmod");    DDUMP( "config_interface/boxes/Box.pmod" );    dump( "base_server/html.pike");       add_constant( "RoxenModule", RoxenModule);    add_constant( "ModuleInfo", ModuleInfo );       add_constant( "load", load);   
Roxen.git/server/base_server/roxen.pike:5117:   //int s = gethrtime();    _configuration = (program)"configuration";    dump( "base_server/configuration.pike" );    dump( "base_server/rxmlhelp.pike" );       // Override the one from prototypes.pike    add_constant( "Configuration", _configuration );   //report_debug( "[Configuration: %.2fms] ", (gethrtime()-s)/1000.0);   }    - mixed get_locale( ) + string get_locale( )   {    return locale->get();   }      int set_u_and_gid (void|int from_handler_thread)   //! Set the uid and gid to the ones requested by the user. If the   //! sete* functions are available, and the define SET_EFFECTIVE is   //! enabled, the euid and egid is set. This might be a minor security   //! hole, but it will enable roxen to start CGI scripts with the   //! correct permissions (the ones the owner of that script have).
Roxen.git/server/base_server/roxen.pike:5567: Inside #if undefined(__NT__)
  #ifndef __NT__    if(getuid())    {    report_debug("It is impossible to chroot() if the server is not run as root.\n");    return;    }       if(!chroot(to))    {    report_debug("Roxen: Cannot chroot to "+to+": "); - #if efun(real_perror) + #if constant(real_perror)    real_perror();   #endif    return;    }    report_debug("Root is now "+to+".\n");   #endif   }      void create_pid_file(string where)   {   #ifndef __NT__    if(!where) return;   // where = replace(where, ({ "$pid", "$uid" }),   // ({ (string)getpid(), (string)getuid() }));       object privs = Privs("Deleting old pid file.");    r_rm(where);    privs = 0; -  if(mixed err = catch { +  +  mixed err; +  +  // Note: The server lock file is often created by the start script, but +  // there is a race, so this code is here for paranoia reasons. +  if (!Stdio.exist(sprintf("/var/run/roxen-server.%d.pid", getpid())) && +  !Stdio.exist(sprintf("/tmp/roxen-server.%d.pid", getpid()))) { +  // NB: The following won't work if there's a wrapper process +  // for Roxen (eg started via gdb, truss or valgrind), +  // but that shouldn't matter much, since the pid lock file +  // won't be used in that case anyway. +  privs = Privs("Creating pid lock."); +  if (catch { +  // Try /var/run/ first. +  hardlink(sprintf("/var/run/roxen-start.%d.pid", getppid()), +  sprintf("/var/run/roxen-server.%d.pid", getpid())); +  } && (err = catch { +  // And then /tmp/. +  hardlink(sprintf("/tmp/roxen-start.%d.pid", getppid()), +  sprintf("/tmp/roxen-server.%d.pid", getpid())); +  })) { +  report_debug("Cannot create the pid lock file %O: %s", +  sprintf("/tmp/roxen-server.%d.pid", getpid()), +  describe_error(err)); +  } +  privs = 0; +  } +  if(err = catch {    Stdio.write_file(where, sprintf("%d\n%d\n", getpid(), getppid()));    })    report_debug("Cannot create the pid file %O: %s",    where, describe_error (err));   #endif   }      Pipe.pipe shuffle(Stdio.File from, Stdio.File to,    Stdio.File|void to2,    function(:void)|void callback)   { - #if efun(spider.shuffle) + #if constant(spider.shuffle)    if(!to2)    {    object p = fastpipe( );    p->input(from);    p->set_done_callback(callback);    p->output(to);    return p;    } else {   #endif    // 'fastpipe' does not support multiple outputs.    Pipe.pipe p = Pipe.pipe();    if (callback) p->set_done_callback(callback);    p->output(to);    if(to2) p->output(to2);    p->input(from);    return p; - #if efun(spider.shuffle) + #if constant(spider.shuffle)    }   #endif   }    -  + // Dump a single thread. + void describe_thread (Thread.Thread thread) + { +  int hrnow = gethrtime(); +  string thread_descr = ""; +  if (string th_name = thread_name(thread, 1)) +  thread_descr += " - " + th_name; +  if (int start_hrtime = thread_task_start_times[thread]) +  thread_descr += sprintf (" - busy for %.3fs", +  (hrnow - start_hrtime) / 1e6); +  report_debug(">> ### Thread 0x%x%s:\n", +  thread->id_number(), +  thread_descr); +  // Use master()->describe_backtrace to sidestep the background +  // failure wrapper that's active in RUN_SELF_TEST. +  string th_bt = master()->describe_backtrace (thread->backtrace()); +  +  // Expand any occurrences of: +  // Thread.Mutex(/*locked by 0x....*/) +  // to: +  // Thread.Mutex(/*locked by 0x.... - <thread name>*/) +  string bt_separator = "Thread.Mutex(/*locked by "; +  if (has_value(th_bt, bt_separator)) { +  array(string) bt_segs = th_bt / bt_separator; +  if (sizeof(bt_segs) > 1) { +  foreach (bt_segs; int idx; string bt_seg) { +  if (sscanf(bt_seg, "0x%[0-9a-fA-F]*/", string th_hex_addr)) { +  if (string th_name = thread_name_from_addr("0x" + th_hex_addr)) { +  bt_segs[idx] = +  "0x" + th_hex_addr + " - " + th_name + +  bt_seg[sizeof(th_hex_addr) + 2..]; +  } +  } +  } +  th_bt = bt_segs * bt_separator; +  } +  } +  +  report_debug(">> " + replace (th_bt, "\n", "\n>> ") + "\n"); + } +    // Dump all threads to the debug log.   void describe_all_threads (void|int ignored, // Might be the signal number. -  void|object threads_disabled) +  void|int(0..1) inhibit_threads_disabled)   { -  if (!threads_disabled) +  object threads_disabled; +  if (!inhibit_threads_disabled)    // Disable all threads to avoid potential locking problems while we    // have the backtraces. It also gives an atomic view of the state.    threads_disabled = _disable_threads();       array(Thread.Thread) threads = all_threads();       report_debug("###### Describing all %d pike threads:\n>>\n",    sizeof (threads));       threads = Array.sort_array (
Roxen.git/server/base_server/roxen.pike:5646:    lambda (Thread.Thread a, Thread.Thread b) {    // Backend thread first, otherwise in id order.    if (a == backend_thread)    return 0;    else if (b == backend_thread)    return 1;    else    return a->id_number() > b->id_number();    });    -  int hrnow = gethrtime(); +     foreach (threads, Thread.Thread thread) { -  string thread_descr = ""; -  if (string th_name = thread_name(thread, 1)) -  thread_descr += " - " + th_name; -  if (int start_hrtime = thread_task_start_times[thread]) -  thread_descr += sprintf (" - busy for %.3fs", -  (hrnow - start_hrtime) / 1e6); -  report_debug(">> ### Thread 0x%x%s:\n", -  thread->id_number(), -  thread_descr); -  // Use master()->describe_backtrace to sidestep the background -  // failure wrapper that's active in RUN_SELF_TEST. -  report_debug(">> " + -  replace (master()->describe_backtrace (thread->backtrace()), -  "\n", "\n>> ") + -  "\n"); +  describe_thread (thread);    }    -  +  threads = 0; +  +  if (catch {    array(array) queue = handle_queue->peek_array();       // Ignore the handle thread shutdown marker, if any.    queue -= ({0});       if (!sizeof (queue)) -  report_debug ("###### No entries in the handler queue\n"); +  report_debug("###### No entries in the handler queue.\n");    else { -  report_debug ("###### %d entries in the handler queue:\n>>\n", +  report_debug("###### %d entries in the handler queue:\n>>\n",    sizeof (queue)); -  foreach (queue; int i; array task) -  report_debug (">> %d: %s\n", i, +  foreach(queue; int i; array task) +  report_debug(">> %d: %s\n", i,    replace (debug_format_queue_task (task), "\n", "\n>> ")); -  report_debug (">> \n"); +  report_debug(">> \n");    } -  +  queue = 0; +  }) { +  report_debug("###### Handler queue busy.\n"); +  }    -  queue = bg_queue->peek_array(); +  if (catch { +  array queue = bg_queue->peek_array();       if (!sizeof (queue))    report_debug ("###### No entries in the background_run queue\n");    else {    report_debug ("###### %d entries in the background_run queue:\n>>\n",    sizeof (queue));    foreach (queue; int i; array task)    report_debug (">> %d: %s\n", i,    replace (debug_format_queue_task (task), "\n", "\n>> "));    report_debug (">> \n");    } -  +  queue = 0; +  }) { +  report_debug("###### background_run queue busy.\n"); +  }       report_debug ("###### Thread and queue dumps done at %s\n", ctime (time()));    -  queue = 0; -  threads = 0; +     threads_disabled = 0;      #ifdef DEBUG    report_debug (RoxenDebug.report_leaks());   #endif   }         // Dump threads by file polling.   
Roxen.git/server/base_server/roxen.pike:5794: Inside #if defined(TIMERS)
   sort( a, b );    reverse(a);    reverse(b);    report_notice("Timers:\n");    for( int i = 0; i<sizeof(b); i++ )    report_notice( " %-30s : %10.1fms\n", b[i], a[i]/1000.0 );    report_notice("\n\n");   }   #endif    + void scan_certs(int|void force) + { +  foreach(query("CertGlobs"), string glob_pattern) { +  glob_pattern = String.trim_all_whites(glob_pattern); +  if (glob_pattern == "") continue; +  if (!has_value(glob_pattern, "*") && !has_value(glob_pattern, "?")) { +  CertDB.register_pem_file(glob_pattern); +  continue; +  } +  string dir = dirname(glob_pattern); +  string base = basename(glob_pattern); +  array(string) dirs = ({}); +  if (has_value(dir, "*") || has_value(dir, "?")) { +  // FIXME: Complicated case; expand the globbed dir. +  dirs = ({ dir }); +  } else { +  dirs = ({ dir }); +  } +  foreach(dirs, dir) { +  array(string) rdirs; +  if (has_prefix(dir, "/")) { +  // Absolute path. +  rdirs = ({ "/" }); +  } else { +  // lopen path +  rdirs = map(roxenloader.package_directories, roxen_path); +  } +  foreach(rdirs, string rdir) { +  array(string) paths = get_dir(combine_path(rdir, dir)); +  if (!paths) { + #ifdef SSL3_DEBUG +  if (errno() != System.ENOENT) { +  werror("Reading PEM dir %O failed: %s\n", +  combine_path(rdir, dir), strerror(errno())); +  } + #endif +  continue; +  } +  foreach(glob(base, paths), string fname) { + #ifdef SSL3_DEBUG +  werror("Found PEM file %O, matching %O.\n", +  Stdio.append_path(dir, fname), glob_pattern); + #endif +  CertDB.register_pem_file(Stdio.append_path(dir, fname)); +  } +  } +  } +  } +  CertDB.refresh_all_pem_files(force); + }    -  + protected BackgroundProcess scan_certs_process; +  + // Start a background process that scan for new certs every 10 minutes. + protected void start_scan_certs() + { +  if (scan_certs_process) return; +  +  scan_certs_process = BackgroundProcess(600, scan_certs); + } +  + protected void stop_scan_certs() + { +  if (scan_certs_process) { +  scan_certs_process->stop(); +  scan_certs_process = UNDEFINED; +  } + } +    protected class GCTimestamp   {    array self_ref;    protected void create() {self_ref = ({this_object()});}    protected void destroy() {    werror ("GC runs at %s", ctime(time()));    GCTimestamp();    }   }    -  + int log_gc_timestamps; + int log_gc_histogram; + int log_gc_verbose; + int log_gc_cycles;    -  + string format_cycle (array(mixed) cycle) + { +  array(string) string_parts = ({}); +  foreach (cycle; int pos; mixed val) { +  string formatted; +  +  if (arrayp (val)) { +  formatted = sprintf ("array(%d)", sizeof (val)); +  } else if (mappingp (val)) { +  formatted = sprintf ("mapping(%d)", sizeof (val)); +  } else if (multisetp (val)) { +  formatted = sprintf ("multiset(%d)", sizeof (val)); +  } else { +  formatted = sprintf ("%O", val); +  } +  +  /* Identify object/mapping/array index of the next element in the cycle. */ +  mixed next_val; +  if (pos < sizeof (cycle) - 1) { +  next_val = cycle[pos + 1]; +  } else { +  next_val = cycle[0]; +  } +  +  if (multisetp(val)) { +  formatted += "[[index]]"; +  } else { +  // NB: This catch is to handle the case where val is an object +  // that implements lfun::_indices() and/or lfun::_values() +  // that throw errors. +  if (catch { +  array(mixed) inds = indices(val); +  array(mixed) vals = values(val); +  int i = search(vals, next_val); +  if (i >= 0) { +  // Found. +  if (intp(inds[i])) { +  formatted += sprintf("[%d]", inds[i]); +  } else if (stringp(inds[i])) { +  if (sizeof(inds[i]) < 100) { +  formatted += sprintf("[%O]", inds[i]); +  } else { +  formatted += sprintf("[string(len: %d)]", sizeof(inds[i])); +  } +  } else { +  formatted += sprintf("[%t]", inds[i]); +  } +  } else { +  i = search(inds, next_val); +  if (i >= 0) { +  formatted += "[[index]]"; +  } else if (objectp(val)) { +  formatted += "->protected"; +  } +  } +  }) { +  formatted += "[[broken]]"; +  } +  } +  +  string_parts += ({ formatted }); +  } +  +  return string_parts * " ==> "; + } +  + void reinstall_gc_callbacks() + { +  mapping(string:mixed) gc_params = ([ "pre_cb": 0, +  "post_cb": 0, +  "destruct_cb": 0, +  "done_cb": 0 ]); +  +  int gc_start; +  +  // mapping from program name (as reported by sprintf/%O) to number of +  // GC-destructed objects. Only valid in the GC's done_cb below. +  mapping(string:int) gc_histogram = ([]); +  +  // mapping from program name (as reported by sprintf/%O) to flag +  // indicating whether a cycle has been reported for this program in +  // the current GC report round. Cleared on every GC restart. +  mapping(string:int(0..1)) reported_cycles = ([]); +  +  if (log_gc_timestamps || log_gc_histogram || log_gc_verbose || +  log_gc_cycles) { +  gc_params->pre_cb = +  lambda() { +  gc_start = gethrtime(); +  gc_histogram = ([]); +  reported_cycles = ([]); +  werror("GC runs at %s", ctime(time())); +  }; +  +  gc_params->post_cb = +  lambda() { +  werror("GC done after %dms\n", +  (gethrtime() - gc_start) / 1000); +  }; +  +  if (log_gc_histogram || log_gc_verbose || log_gc_cycles) { +  gc_params->destruct_cb = +  lambda(object o) { +  // NB: These calls to sprintf(%O) can +  // take significant time. +  string id = +  sprintf("%O", object_program(o)); +  gc_histogram[id]++; +  if (log_gc_verbose) { +  werror("GC cyclic reference in %O.\n", +  o); +  } +  +  if (log_gc_cycles && !reported_cycles[id]) { +  reported_cycles[id] = 1; +  if (array(mixed) cycle = Pike.identify_cycle(o)) { +  werror ("GC cycle:\n%s\n", format_cycle (cycle)); +  } +  } +  }; +  } +  +  gc_params->done_cb = +  lambda(int n) { +  if (!n) return; +  werror("GC zapped %d things.\n", n); +  +  if (log_gc_histogram) { +  mapping h = gc_histogram; +  gc_histogram = ([]); +  if (!sizeof(h)) return; +  array i = indices(h); +  array v = values(h); +  sort(v, i); +  werror("GC histogram:\n"); +  foreach(reverse(i)[..9], string p) { +  werror("GC: %s: %d\n", p, h[p]); +  } +  } +  }; +  } +  +  Pike.gc_parameters(gc_params); + } +    array argv;   int main(int argc, array tmp)   {    // __builtin.gc_parameters((["enabled": 0]));    argv = tmp;    tmp = 0;      #if 0    Thread.thread_create (lambda () {    while (1) {    sleep (10);    describe_all_threads();    }    });   #endif    -  +  +    #ifdef LOG_GC_TIMESTAMPS -  GCTimestamp(); +  log_gc_timestamps = 1;   #endif -  + #ifdef LOG_GC_HISTOGRAM +  log_gc_histogram = 1; + #endif + #ifdef LOG_GC_VERBOSE +  log_gc_verbose = 1; + #endif + #ifdef LOG_GC_CYCLES +  log_gc_cycles = 1; + #endif    -  +  reinstall_gc_callbacks(); +     // For RBF    catch(mkdir(getenv("VARDIR") || "../var"));       dbm_cached_get = master()->resolv( "DBManager.cached_get" );       dbm_cached_get( "local" )->    query( "CREATE TABLE IF NOT EXISTS "    "compiled_formats ("    " md5 CHAR(32) not null primary key,"    " full BLOB not null,"    " enc BLOB not null"    ")" );    master()->resolv( "DBManager.is_module_table" )    ( 0, "local", "compiled_formats",    "Compiled and cached log and security pattern code. "); -  +  master()->resolv( "DBManager.inhibit_backups" ) +  ( "local", "compiled_formats", );       slowpipe = ((program)"base_server/slowpipe");    fastpipe = ((program)"base_server/fastpipe");    dump( "etc/modules/DBManager.pmod" );    dump( "etc/modules/VFS.pmod" );    dump( "base_server/slowpipe.pike" );    dump( "base_server/fastpipe.pike" );    dump( "base_server/throttler.pike" );       if (!has_value (compat_levels, roxen_ver))    report_debug ("Warning: The current version %s does not exist in "    "roxen.compat_levels.\n", roxen_ver);       add_constant( "Protocol", Protocol );   #ifdef TIMERS    call_out( show_timers, 30 );   #endif    - #if constant(SSL.sslfile) + #if constant(SSL.File) +  add_constant( "StartTLSProtocol", StartTLSProtocol );    add_constant( "SSLProtocol", SSLProtocol ); -  +  +  dbm_cached_get("roxen")-> +  query("CREATE TABLE IF NOT EXISTS cert_pem_files (" +  " id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, " +  // lopen()-compatible path to the PEM file. +  " path VARCHAR(2047) NOT NULL, " +  // Password to decode the PEM data (if any). +  " pass VARCHAR(255) NULL, " +  // mtime for the PEM file at last scan. +  // NULL if not valid. +  " mtime INT NULL, " +  // time at which the PEM file was last imported. +  // NULL if not imported yet. +  " itime INT NULL, " +  // Hash (currently SHA256) of PEM file data at last scan. +  // NULL if not imported. +  " hash VARBINARY(64) NULL, " +  // Index used when (un-)registering PEM files. +  " INDEX path (path), " +  // Index used when rescanning PEM files. +  " INDEX itime (itime)" +  ")"); +  master()->resolv( "DBManager.is_module_table" ) +  ( 0, "roxen", "certs", "Registry of known PEM files."); +  +  dbm_cached_get("roxen")-> +  query("CREATE TABLE IF NOT EXISTS certs (" +  " id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, " +  // Distinguished Name for the certified subject. +  " subject VARBINARY(255) NOT NULL, " +  // DN for the issuer of this certificate. +  " issuer VARBINARY(255) NOT NULL, " +  // Id for the cert that this cert is issued by. +  // NULL for self-signed or well known. +  " parent INT NULL, " +  // Id of the source PEM file. +  // NULL if stale. +  " pem_id INT NULL, " +  // Message number in the PEM file. +  // NULL if stale or refresh in progress. +  " msg_no INT DEFAULT 0 NULL, " +  // Expiry timestamp for the certificate. +  " expires INT NOT NULL, " +  // Data contained in the PEM. +  " data BLOB NOT NULL, " +  // Public key hash. +  " keyhash VARBINARY(64) NOT NULL, " +  // Index used when refreshing a PEM file. +  " INDEX (pem_id, msg_no), " +  // Index used when searching for certs matching a key. +  " INDEX keyhash (keyhash), " +  // Index used when searching for issuers and refreshing the cert. +  " INDEX subject (subject)," +  // Index used when searching for signed entities when +  // refreshing the cert. +  " INDEX issuer (issuer)" +  ")"); +  master()->resolv( "DBManager.is_module_table" ) +  ( 0, "roxen", "certs", "SSL/TLS Certificates."); +  +  dbm_cached_get("roxen")-> +  query("CREATE TABLE IF NOT EXISTS cert_keys (" +  " id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, " +  // Id of the source PEM file. +  // NULL if stale. +  " pem_id INT NULL, " +  // Message number in the PEM file. +  // NULL if stale or refresh in progress. +  " msg_no INT DEFAULT 0 NULL, " +  // Public key hash. +  // NULL if PEM decryption unsuccessful. +  " keyhash VARBINARY(64) NULL, " +  // Encrypted private key ASN.1. +  // Encrypted with AES.CCM keyed with SHA256(cert_secret + keyhash). +  // NULL if PEM decryption unsuccessful. +  " data BLOB NULL, " +  // Index used when refreshing a PEM file. +  " INDEX (pem_id, msg_no), " +  // Index used when searching for keys matching a cert. +  " INDEX keyhash (keyhash)" +  ")"); +  master()->resolv( "DBManager.is_module_table" ) +  ( 0, "roxen", "cert_keys", "SSL/TLS Private Keys."); +  +  dbm_cached_get("roxen")-> +  query("CREATE TABLE IF NOT EXISTS cert_keypairs (" +  " id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, " +  // Id for the cert. +  " cert_id INT NOT NULL, " +  // Id for the corresponding key. +  " key_id INT NOT NULL, " +  // Display name for the keypair. +  " name VARCHAR(255) NOT NULL DEFAULT '', " +  " INDEX (cert_id, key_id)" +  ")"); +  master()->resolv( "DBManager.is_module_table" ) +  ( 0, "roxen", "cert_keys", "SSL/TLS Key and Certificate matching.");   #endif       dump( "etc/modules/Variable.pmod/module.pmod" );    dump( "etc/modules/Variable.pmod/Language.pike" );    dump( "etc/modules/Variable.pmod/Schedule.pike" );       foreach( glob("*.pike", get_dir( "etc/modules/Variable.pmod/"))    -({"Language.pike", "Schedule.pike"}), string f )    DDUMP( "etc/modules/Variable.pmod/"+f );   
Roxen.git/server/base_server/roxen.pike:5921:    argc = sizeof(argv);       fonts = ((program)"base_server/fonts.pike")();       DDUMP( "languages/abstract.pike" );    initiate_languages(query("locale"));       cache_clear_deltas();    set_locale();    - #if efun(syslog) + #if constant(syslog)    init_logger();   #endif    init_garber();       initiate_supports();    initiate_argcache();    init_configuserdb();    cache.init_session_cache();       protocols = build_protocols_mapping();
Roxen.git/server/base_server/roxen.pike:5967: Inside #if defined(THREADS)
   backend_thread = this_thread();   #ifdef THREADS    name_thread( backend_thread, "Backend" );   #else    report_debug("\n"    "WARNING: Threads not enabled!\n"    "\n");   #endif /* THREADS */       foreach(({ "testca.pem", "demo_certificate.pem" }), string file_name) { -  if (sizeof(roxenloader.package_directories) && -  (lfile_path(file_name) == file_name)) { -  file_name = roxen_path (roxenloader.package_directories[-1] + "/" + +  if (!sizeof(roxenloader.package_directories)) break; +  CertDB.register_pem_file(file_name); +  string cert; +  if (lfile_path(file_name) == file_name) { +  file_name = roxen_path (roxenloader.package_directories[0] + "/" +    file_name);    report_notice("Generating a new certificate %s...\n", file_name); -  string cert = Roxen.generate_self_signed_certificate("*"); +  cert = Roxen.generate_self_signed_certificate("*"); + #if constant(Standards.X509) +  } else { +  file_name = roxen_path (lfile_path(file_name));    -  +  // Check if we need to upgrade the cert. +  // +  // Certificates generated by old versions of Pike were +  // plain X.509v1, while certificates generated by Pike 8.0 +  // and later are X.509v3 with some required extensions. +  +  string old_cert = Stdio.read_bytes(file_name); +  if (!old_cert) { +  report_error("Failed to read certificate %s.\n", file_name); +  continue; +  } +     // Note: set_u_and_gid() hasn't been called yet,    // so there's no need for Privs. -  +  Standards.PEM.Messages msgs = Standards.PEM.Messages(old_cert); +  +  int upgrade_needed; +  +  foreach(msgs->parts; string part; array(Standards.PEM.Message) msg) { +  if (!has_suffix(part, "CERTIFICATE")) continue; +  Standards.X509.TBSCertificate tbs = +  Standards.X509.decode_certificate(msg[0]->body); +  upgrade_needed = (tbs->version < 3); +  if (!upgrade_needed) { +  // NB: This stuff is not only to work around that +  // Standards.X509.algorithms is protected, but +  // also to avoid having to know about how the +  // X509 algorithm sequence is structured. +  class HashAlgVerifier { +  inherit Standards.X509.Verifier; +  constant type = "hash"; +  class HashPKC { +  inherit Crypto.Sign.State; +  int(0..1) pkcs_verify(string(8bit) msg, +  Crypto.Hash h, +  string(8bit) sign) +  { +  // Disallow SHA1 and shorter. +  // Allow SHA224 and longer. +  return h && (h->digest_size() >= 28); +  } +  }; +  protected void create() +  { +  pkc = HashPKC(); +  } +  }; +  upgrade_needed = !HashAlgVerifier()->verify(tbs->algorithm, "", ""); +  } +  break; +  } +  +  if (!upgrade_needed || (sizeof(msgs->parts) != 2)) continue; +  +  // NB: We reuse the old key. +  Crypto.Sign key; +  foreach(msgs->parts; string part; array(Standards.PEM.Message) msg) { +  if (!has_suffix(part, "PRIVATE KEY")) continue; +  if (msg[0]->headers["dek-info"]) { +  // Not supported here. +  break; +  } +  key = Standards.X509.parse_private_key(msg[0]->body); +  } +  if (!key) continue; +  +  report_notice("Renewing certificate: %O...\n", file_name); +  cert = Roxen.generate_self_signed_certificate("*", key); + #endif /* constant(Standards.X509) */ +  } +  +  if (cert) { +  // Note: set_u_and_gid() hasn't been called yet, +  // so there's no need for Privs.    Stdio.File file = Stdio.File(); -  if (!file->open(file_name, "wxc", 0600)) { +  if (!file->open(file_name, "wtc", 0600)) {    report_error("Couldn't create certificate file %s: %s\n", file_name,    strerror (file->errno()));    } else if (file->write(cert) != sizeof(cert)) { -  rm(cert); +  rm(file_name);    report_error("Couldn't write certificate file %s: %s\n", file_name,    strerror (file->errno()));    }    }    }    -  +  // Update the certificate registry before opening any ports. +  // NB: Force all certificate files to be reread and reparsed. +  scan_certs(1); +     enable_configurations();       string pid_file = Getopt.find_option(argv, "p", "pid-file");    if (pid_file && query("permanent_uid")) rm(pid_file);       set_u_and_gid(); // Running with the right [e]uid:[e]gid from this point on.       create_pid_file(pid_file);       // Done before the modules are dumped.
Roxen.git/server/base_server/roxen.pike:6012:    if( Getopt.find_option( argv, 0, "no-delayed-load" ))    enable_configurations_modules();    else    foreach( configurations, Configuration c )    if( c->query( "no_delayed_load" ) )    c->enable_all_modules();   #endif // RUN_SELF_TEST      #ifdef THREADS    start_handler_threads(); + #if constant(Filesystem.Monitor.basic) +  start_fsgarb(); + #endif   #endif /* THREADS */    -  +  start_scan_certs(); +  start_hourly_maintenance(); +    #ifdef TEST_EUID_CHANGE    if (test_euid_change) {    Stdio.File f = Stdio.File();    if (f->open ("rootonly", "r") && f->read())    werror ("Backend thread can read rootonly\n");    else    werror ("Backend thread can't read rootonly\n");    }   #endif    -  +  // Signals which need to be ignored +  foreach( ({ "SIGPIPE" }), string sig) +  catch( signal(signum(sig), 0) ); +     // Signals which cause a restart (exitcode != 0)    foreach( ({ "SIGINT", "SIGTERM" }), string sig)    catch( signal(signum(sig), async_sig_start(exit_when_done,0)) );       catch(signal(signum("SIGHUP"),async_sig_start(reload_all_configurations,1)));       // Signals which cause Roxen to dump the thread state    foreach( ({ "SIGBREAK", "SIGQUIT",   #ifdef ENABLE_SIGWINCH    "SIGWINCH",
Roxen.git/server/base_server/roxen.pike:6158:   // Called from the administration interface.   string check_variable(string name, mixed value)   {    switch(name)    {   #ifndef __NT__    case "abs_engage":    if (value)    // Make sure restart_if_stuck is called from the backend thread.    call_out(restart_if_stuck, 0, 1); -  else +  else {    remove_call_out(restart_if_stuck); -  +  alarm(0); +  }    break;    case "abs_timeout":    if (value < 0) {    return "The timeout must be >= 0 minutes.";    }    break;   #endif       case "suicide_schedule":    case "suicide_engage":
Roxen.git/server/base_server/roxen.pike:6338:    }       protected string extract_user(string from)    {    array tmp;    if (!from || sizeof(tmp = from/":")<2)    return "-";    return tmp[0]; // username only, no password    }    +  protected string get_forwarded_field(RequestID id, string field) +  { +  foreach(id->misc->forwarded || ({}), array(string|int) segment) { +  if (!arrayp(segment) || sizeof(segment) < 3) continue; +  if (segment[0] != field || segment[1] != '=') continue; +  return MIME.quote(segment[2..]); +  } +  return "-"; +  } +     void log_access( function do_write, RequestID id, mapping file );       void log_event (function do_write, string facility, string action,    string resource, mapping(string:mixed) info);       protected void do_async_write( string host, string data,    string ip, function c )    {    if( c )    c( replace( data, "\4711", (host||ip) ) );
Roxen.git/server/base_server/roxen.pike:6404:       // Used for access logging    "host": ({"\4711" /* unlikely to occur normally */, 0,    1, "\"-\"", LOG_ASYNC_HOST}),    "vhost": ({"%s", "(request_id->misc->host||\"-\")",    1, "\"-\"", 0}),    "ip-number": ({"%s", "(string)request_id->remoteaddr",    1, "\"0.0.0.0\"", 0}),    "bin-ip-number": ({"%s", "host_ip_to_int(request_id->remoteaddr)",    1, "\"\\0\\0\\0\\0\"", 0}), +  "link-layer": ({"%s", "request_id->query_link_layer()", +  1, "\"-\"", 0}), +  "cipher-suite": ({"%s", "request_id->query_cipher_suite()", +  1, "\"-\"", 0}),    "method": ({"%s", "(string)request_id->method",    1, "\"-\"", 0}),    "full-resource": ({"%s", ("(string)"    "(request_id->raw_url||"    " (request_id->misc->common&&"    " request_id->misc->common->orig_url)||"    " request_id->not_query)"),    "%s" , "url_encode (resource)", 0}),    "cs-uri-stem": ({"%s", ("(string)"    "((request_id->misc->common&&"
Roxen.git/server/base_server/roxen.pike:6436:    "%s" , "url_encode (resource)", 0}),    "real-full-resource": ({"%s", ("(string)(request_id->raw_url||"    " request_id->not_query)"),    "%s" , "url_encode (resource)", 0}),    "real-cs-uri-stem": ({"%s", ("(string)(request_id->not_query||"    " (request_id->raw_url && "    " (request_id->raw_url/\"?\")[0])||"    " \"-\")"),    "%s" , "url_encode (resource)", 0}),    "protocol": ({"%s", "(string)request_id->prot", 1, "\"-\"", 0}), +  "scheme": ({"%s", "(string)((request_id->port_obj && " +  "request_id->port_obj->prot_name) || \"-\")", +  1, "\"-\"", 0 }),    "response": ({"%d", "(int)(file->error || 200)", 1, "\"-\"", 0}),    "bin-response": ({"%2c", "(int)(file->error || 200)", 1, "\"\\0\\0\"", 0}),    "length": ({"%d", "(int)file->len", 1, "\"0\"", 0}),    "bin-length": ({"%4c", "(int)file->len", 1, "\"\\0\\0\\0\\0\"", 0}),    "queue-length": ({"%d", "(int) request_id->queue_length",    1, "\"-\"", 0}),    "request-time": ({"%1.4f", ("(float)(gethrtime() - "    " request_id->hrtime) /"    "1000000.0"),    1, "\"-\"", 0}),
Roxen.git/server/base_server/roxen.pike:6495:    "\"0\""),    1, "\"-\"", 0}),    "content-type": ({"%s", ("(((arrayp(file->type) ? "    "file->type[0] : file->type) "    "|| \"-\") / \";\")[0]"),    1, "\"-\"", 0}),    "cookies": ({"%s", ("arrayp(request_id->request_headers->cookie)?"    "request_id->request_headers->cookie*\";\":"    "request_id->request_headers->cookie||\"\""),    1, "\"-\"", 0}), +  "set-cookies": ({"%s", ("Array.uniq(" +  "request_id->get_response_headers(\"Set-Cookie\")+" +  "request_id->get_response_headers(\"set-cookie\"))" +  "*\";\""), +  1, "\"-\"", 0}),    "cache-status": ({"%s", ("sizeof(request_id->cache_status||({}))?"    "indices(request_id->cache_status)*\",\":"    "\"nocache\""),    1, "\"-\"", 0}),    "eval-status": ({"%s", ("sizeof(request_id->eval_status||({}))?"    "indices(request_id->eval_status)*\",\":"    "\"-\""),    1, "\"-\"", 0}),    "protcache-cost": ({"%d", "request_id->misc->protcache_cost",    1, "\"-\"", 0}), -  "xff": ({"%s", ("arrayp(request_id->request_headers[" -  "\"x-forwarded-for\"]) ? " -  "(request_id->request_headers[" -  "\"x-forwarded-for\"][-1] / \",\")[0] :" -  "((request_id->request_headers[" -  "\"x-forwarded-for\"] || \"-\") / \",\")[0]"), +  "forwarded": ({"%s", ("request_id->misc->forwarded ? " +  "MIME.quote(request_id->misc->forwarded *" +  " ({ ',' })) : \"-\""),    1, "\"-\"", 0 }), -  +  "xff": ({"%s", "get_forwarded_field(request_id, \"for\")", +  1, "\"-\"", 0 }),       // Used for event logging    "facility": ({"-", 0, "%s", "facility", 0}),    "action": ({"-", 0, "%s", "action", 0}),   ]);      void run_log_format( string fmt, function c, RequestID id, mapping file )   {    (compiled_log_access[ fmt ] ||    compile_log_format( fmt )->log_access) (c,id,file);
Roxen.git/server/base_server/roxen.pike:6841: Inside #if defined(SECURITY_PATTERN_DEBUG) || defined(HTACCESS_DEBUG)
   },   #if defined(SECURITY_PATTERN_DEBUG) || defined(HTACCESS_DEBUG)    " report_debug(sprintf(\"Verifying against IP %%O (0x%%08x).\\n\",\n"    " id->remoteaddr, remote_ip));\n"   #endif /* SECURITY_PATTERN_DEBUG || HTACCESS_DEBUG */    " if (%s)",    (< " int remote_ip = Roxen.ip_to_int(id->remoteaddr)" >),    }), "ip", }),    ({ "user=%s",1,({ 1,    lambda( string x ) { -  return ({sprintf("(< %{%O, %}>)", x/"," )}); +  return ({sprintf("((multiset)(< %{%O, %}>))", x/"," )});    },       " if (((user || (user = authmethod->authenticate(id, userdb_module)))\n"    " && ((%[0]s->any) || (%[0]s[user->name()]))) || %[0]s->ANY) ",    (<" User user" >),    // No need to NOCACHE () here, since it's up to the    // auth-modules to do that.    }), "user", }),    ({ "group=%s",1,({ 1,    lambda( string x ) { -  return ({sprintf("(< %{%O, %}>)", x/"," )}); +  return ({sprintf("((multiset)(< %{%O, %}>))", x/"," )});    },    " if ((user || (user = authmethod->authenticate(id, userdb_module)))\n"    " && ((%[0]s->any && sizeof(user->groups())) ||\n"    " sizeof(mkmultiset(user->groups())&%[0]s)))",    (<" User user" >),    // No need to NOCACHE () here, since it's up to the    // auth-modules to do that.    }), "group", }),    ({ "dns=%s",1,({    " if(!dns && \n"
Roxen.git/server/base_server/roxen.pike:6964:   //! return, only useful for 'allow'.   //!   //! 'deny' always implies a return, no futher testing is done if a   //! 'deny' match.   {    // Now, this cache is not really all that performance critical, I    // mostly coded it as a proof-of-concept, and because it was more    // fun that trying to find the bug in the image-cache at the moment.       // Note similar code in compile_log_format. -  +  if (pattern == "") +  return 0;    string kmd5 = md5( pattern );      #if !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG)    array tmp =    dbm_cached_get( "local" )    ->query("SELECT full,enc FROM compiled_formats WHERE md5=%s", kmd5 );       if( sizeof(tmp) && (tmp[0]->full == pattern) )    {    mixed err = catch {
Roxen.git/server/base_server/roxen.pike:7348:    compressor_process = Process.Process(({ compressor_program,    compress_file }));    return;    }    }    }       private void do_open_co() { handle(do_open); }    private void do_open(void|object mutex_key)    { -  if (!this_object()) return; // We've been destructed, return +     if (!mutex_key) mutex_key = lock->lock(); -  if (!this_object()) return; // We've been destructed, return +        mixed parent;    if (catch { parent = function_object(object_program(this_object())); } ||    !parent) {    // Our parent (aka the configuration) has been destructed.    // Time to die.    remove_call_out(do_open_co);    remove_call_out(do_close_co);    destruct();    return;
Roxen.git/server/base_server/roxen.pike:7395:    "\n");    return;    }    opened = 1;    remove_call_out(do_open_co);    call_out(do_open_co, 900);    remove_call_out(do_close_co);    call_out(do_close_co, 10.0);    }    -  private void do_close_co() { handle(do_close); } -  private void do_close() +  private void do_close_co() { handle(close); } +  +  void close()    { -  if (!this_object()) return; // We've been destructed, return +     object mutex_key = lock->lock(); -  if (!this_object()) return; // We've been destructed, return +        destruct( fd );    opened = 0;    }       private array(string) write_buf = ({}); -  private void do_the_write_co() { handle(do_the_write); } +     private void do_the_write()    { -  if (!this_object()) return; // We've been destructed, return +     object mutex_key = lock->lock(); -  if (!this_object()) return; // We've been destructed, return +        if (!opened) do_open(mutex_key);    if (!opened) return; -  mixed err = catch (fd->write(write_buf)); +  if (!sizeof (write_buf)) return; +  +  array(string) buf = write_buf; +  // Relying on the interpreter lock here. +  write_buf = ({}); +  +  mixed err = catch (fd->write(buf));    if (err)    catch {    foreach (write_buf, string str)    if (String.width (str) > 8)    werror ("Got wide string in log output: %O\n", str);    }; -  write_buf = ({}); +     remove_call_out(do_close_co);    call_out(do_close_co, 10.0); -  +     if (err)    throw (err);    }       int write( string what )    { -  if (!this_object()) return 0; // We've been destructed, return -  object mutex_key = lock->lock(); -  if (!this_object()) return 0; // We've been destructed, return -  -  if (!sizeof(write_buf)) -  call_out(do_the_write_co, 1); +     write_buf += ({ what }); -  +  if (sizeof (write_buf) == 1) +  background_run (1, do_the_write);    return strlen(what);    }   }