Branch: Tag:

2019-09-16

2019-09-16 13:34:43 by Karl Gustav Sterneberg <kg@roxen.com>

Image cache: Support data larger than 16 MB. [WS-527]

1: - // This file is part of Roxen WebServer. - // Copyright © 1996 - 2009, Roxen IS. - // - // The Roxen WebServer main program. - // - // Per Hedbor, Henrik Grubbström, Pontus Hagland, David Hedbor and others. - // ABS and suicide systems contributed freely by Francesco Chemolli +     - constant cvs_version="$Id$"; -  - //! @appears roxen - //! - //! The Roxen WebServer main program. -  - // The argument cache. Used by the image cache. - ArgCache argcache; -  - // Some headerfiles - #define IN_ROXEN - #include <roxen.h> - #include <config.h> - #include <module.h> - #include <stat.h> - #include <timers.h> -  - // Inherits - inherit "global_variables"; - #ifdef SNMP_AGENT - inherit "snmpagent"; - #endif - #ifdef SMTP_RELAY - inherit "smtprelay"; - #endif - inherit "hosts"; - inherit "disk_cache"; - inherit "fsgc"; - // inherit "language"; - inherit "supports"; - inherit "module_support"; - inherit "config_userdb"; -  - // Used to find out which thread is the backend thread. - Thread.Thread backend_thread; -  - // --- Locale defines --- -  - //<locale-token project="roxen_start"> LOC_S </locale-token> - //<locale-token project="roxen_message"> LOC_M </locale-token> - //<locale-token project="roxen_config"> LOC_C </locale-token> - #define LOC_S(X,Y) _STR_LOCALE("roxen_start",X,Y) - #define LOC_M(X,Y) _STR_LOCALE("roxen_message",X,Y) - #define LOC_C(X,Y) _STR_LOCALE("roxen_config",X,Y) - #define CALL_M(X,Y) _LOCALE_FUN("roxen_message",X,Y) -  - // --- Debug defines --- -  - #ifdef SSL3_DEBUG - # define SSL3_WERR(X) report_debug("TLS port %s: %s\n", get_url(), X) - #else - # define SSL3_WERR(X) - #endif -  - #ifdef THREAD_DEBUG - # define THREAD_WERR(X) report_debug("Thread: "+X+"\n") - #else - # define THREAD_WERR(X) - #endif -  - #ifdef LOG_GC_VERBOSE - #define LOG_GC_HISTOGRAM - #endif -  - #ifdef LOG_GC_HISTOGRAM - #define LOG_GC_TIMESTAMPS - #endif -  - // Needed to get core dumps of seteuid()'ed processes on Linux. - #if constant(System.dumpable) - #define enable_coredumps(X) System.dumpable(X) - #else - #define enable_coredumps(X) - #endif -  - #define DDUMP(X) sol( combine_path( __FILE__, "../../" + X ), dump ) - protected function sol = master()->set_on_load; -  - // Tell Pike.count_memory this is global. - constant pike_cycle_depth = 0; -  - #ifdef TEST_EUID_CHANGE - int test_euid_change; - #endif -  - string md5( string what ) - { -  return Gmp.mpz(Crypto.MD5.hash( what ),256)->digits(32); - } -  - string query_configuration_dir() - { -  return configuration_dir; - } -  - //! @ignore - array(string) query_hot_reload_modules() - //! Returns an array of modules added for hot reloading via - //! @tt{--module-hot-reload=<modname>@}. - { -  if (hot_reload_modules) { -  return map(replace(hot_reload_modules, " ", ",")/",", -  String.trim_all_whites) - ({ "" }); -  } -  -  return ({}); - } -  - array(string) query_hot_reload_modules_conf() - //! Returns an array of modules added for hot reloading via - //! @tt{--module-hot-reload-conf=<conf>@}. - { -  if (hot_reload_modules_conf) { -  return map(replace(hot_reload_modules_conf, " ", ",")/",", -  String.trim_all_whites) - ({ "" }); -  } -  -  return 0; - } - //! @endignore -  - array(string|int) filename_2 (program|object o) - { -  if( objectp( o ) ) -  o = object_program( o ); -  -  string fname = Program.defined (o); -  int line; -  if (fname) { -  array(string) p = fname / ":"; -  if (sizeof (p) > 1 && p[-1] != "" && sscanf (p[-1], "%d%*c", int l) == 1) { -  fname = p[..<1] * ":"; -  line = l; -  } -  } -  -  else if( !fname ) { -  fname = master()->program_name( o ); -  if (!fname) -  return ({0, 0}); -  } -  -  string cwd = getcwd() + "/"; -  if (has_prefix (fname, cwd)) -  fname = fname[sizeof (cwd)..]; -  else if (has_prefix (fname, roxenloader.server_dir + "/")) -  fname = fname[sizeof (roxenloader.server_dir + "/")..]; -  -  return ({fname, line}); - } -  - string filename( program|object o ) - { -  [string fname, int line] = filename_2 (o); -  return fname || "(unknown program)"; - } -  - protected int once_mode; - // String of modules added for hot reloading via --module-hot-reload=<mod> - protected string hot_reload_modules; - protected string hot_reload_modules_conf; -  - // Note that 2.5 is a nonexisting version. It's only used for the - // cache static optimization for tags such as <if> and <emit> inside - // <cache> since that optimization can give tricky incompatibilities - // with 2.4. - // Note also that 5.3 only existed in the Print repository, and - // thus is skipped here. - array(string) compat_levels = ({"2.1", "2.2", "2.4", "2.5", -  "3.3", "3.4", -  "4.0", "4.5", -  "5.0", "5.1", "5.2", "5.4", "5.5", -  "6.0", "6.1", "6.2", - }); -  - // Compat stubs for relocated methods - #ifdef THREADS - string thread_name_from_addr(string hex_addr) - { -  return Roxen.thread_name_from_addr(hex_addr); - } -  - string thread_name(object thread, int|void skip_auto_name) - { -  return Roxen.thread_name(thread, skip_auto_name); - } -  - void name_thread( object thread, string name ) - { -  Roxen.name_thread(thread, name); - } - #endif -  -  - /* Used by read_config.pike, since there seems to be problems with -  * overloading otherwise. -  */ - protected Privs PRIVS(string r, int|string|void u, int|string|void g) - { -  return Privs(r, u, g); - } -  - // Current Configuration. - Thread.Local current_configuration = Thread.Local(); -  - // font cache and loading. - // - // This will be changed to a list of server global modules, to make it - // easier to implement new types of fonts (such as PPM color fonts, as - // an example) - class Fonts - { -  class Font -  { -  Image.Image write( string ... what ); -  array(int) text_extents( string ... what ); -  }; -  array available_font_versions(string name, int size); -  string describe_font_type(string n); -  Font get_font(string f, int size, int bold, int italic, -  string justification, float|int xspace, float|int yspace); -  -  Font resolve_font(string f, string|void justification); -  array(string) available_fonts(int(0..1)|void force_reload); - } - Fonts fonts; -  - // Will replace Configuration after create() is run. - program _configuration; /*set in create*/ -  - array(Configuration) configurations = ({}); -  - private void stop_all_configurations() - { -  configurations->unregister_urls(); - #ifdef THREADS -  // Spend some time waiting out the handler threads before starting -  // to shut down the modules. -  hold_handler_threads(); -  release_handler_threads(3); - #endif -  configurations->stop(1); - } -  - // Function that actually shuts down Roxen. (see low_shutdown). - private void really_low_shutdown(int exit_code) - { -  // Die nicely. Catch for paranoia reasons - #ifdef THREADS -  if (mixed err = catch (stop_handler_threads())) -  werror (describe_backtrace (err)); - #endif /* THREADS */ -  if (!exit_code || once_mode) { -  // We're shutting down; Attempt to take mysqld with us. -  if (mixed err = -  catch { report_notice("Shutting down MySQL.\n"); } || -  catch { -  Sql.Sql db = connect_to_my_mysql(0, "mysql"); -  db->shutdown(); -  }) -  master()->handle_error (err); -  } -  // Zap some of the remaining caches. -  destruct (argcache); -  destruct (cache); - #if 0 -  // Disabled since it's lying when the server is shut down with a -  // SIGTERM or SIGINT to the start script (which include the stop -  // action of the init.d script). -  if (mixed err = catch { -  if (exit_code && !once_mode) -  report_notice("Restarting Roxen.\n"); -  else -  report_notice("Shutting down Roxen.\n"); -  }) -  master()->handle_error (err); - #endif -  roxenloader.real_exit( exit_code ); // Now we die... - } -  - private int shutdown_recurse; -  - // Shutdown Roxen - // exit_code = 0 True shutdown - // exit_code = -1 Restart - private void low_shutdown(int exit_code, int|void apply_patches) - { -  if(shutdown_recurse >= 4) -  { -  if (mixed err = -  catch (report_notice("Exiting roxen (spurious signals received).\n")) || -  catch (stop_all_configurations())) -  master()->handle_error (err); -  // Zap some of the remaining caches. -  destruct(argcache); -  destruct(cache); -  stop_scan_certs(); -  stop_hourly_maintenance(); - #ifdef THREADS - #if constant(Filesystem.Monitor.basic) -  stop_fsgarb(); - #endif -  if (mixed err = catch (stop_handler_threads())) -  master()->handle_error (err); - #endif /* THREADS */ -  roxenloader.real_exit(exit_code); -  } -  if (shutdown_recurse++) return; -  - #ifndef NO_SLOW_REQ_BT -  // Turn off the backend thread monitor while we're shutting down. -  slow_be_timeout_changed(); - #endif -  -  DBManager.stop_backup_thread(); -  -  if ((apply_patches || query("patch_on_restart")) > 0) { -  mixed err = catch { -  foreach(plib->file_list_imported(), mapping(string:mixed) item) { -  report_notice("Applying patch %s...\n", item->metadata->id); -  mixed err = catch { -  plib->install_patch(item->metadata->id, -  "Internal Administrator"); -  }; -  if (err) { -  report_error("Failed to install patch %s: %s\n", -  item->metadata->id, -  describe_backtrace(err)); -  } -  } -  }; -  if (err) { -  master()->handle_error(err); -  } -  } -  -  if (mixed err = catch(stop_all_configurations())) -  master()->handle_error (err); -  - #ifdef SNMP_AGENT -  if(objectp(snmpagent)) { -  snmpagent->stop_trap(); -  snmpagent->disable(); -  } - #endif -  -  call_out(really_low_shutdown, 0.1, exit_code); - } -  - private int shutdown_started; -  - // Perhaps somewhat misnamed, really... This function will close all - // listen ports and then quit. The 'start' script should then start a - // new copy of roxen automatically. - void restart(float|void i, void|int exit_code, void|int apply_patches) - //! Restart roxen, if the start script is running - { -  shutdown_started = 1; -  call_out(low_shutdown, i, exit_code || -1, apply_patches); - } -  - void shutdown(float|void i, void|int apply_patches) - //! Shut down roxen - { -  shutdown_started = 1; -  call_out(low_shutdown, i, 0, apply_patches); - } -  - void exit_when_done() - { -  shutdown_started = 1; -  report_notice("Interrupt request received.\n"); -  low_shutdown(-1, -1); - } -  - int is_shutting_down() - //! Returns true if Roxen is shutting down. - { -  return shutdown_started; - } -  -  - /* -  * handle() stuff -  */ -  - #ifdef THREADS - // function handle = threaded_handle; -  - Thread.Thread do_thread_create(string id, function f, mixed ... args) - { -  Thread.Thread t = thread_create(f, @args); -  Roxen.name_thread( t, id ); -  return t; - } -  - #if 1 - constant Queue = Thread.Queue; - #else - // Shamelessly uses facts about pikes preemting algorithm. - // Might have to be fixed in the future. - class Queue - //! Thread.Queue lookalike, which uses some archaic and less - //! known features of the preempting algorithm in pike to optimize the - //! read function. - // - // If those archaic and less known features are to depend on the - // interpreter lock in the while loop that waits on the condition - // variable then it doesn't work since Pike always might yield before - // a function call (specifically, the wait() call in the condition - // variable). Thus a handler thread might wait even though there is a - // request to process. However, the only effect is that that specific - // request isn't serviced timely; when the next request comes in the - // thread will be woken up and both requests will be handled. - // Furthermore it's extremely rare in the first place since there - // normally are several handler threads. - { -  inherit Thread.Condition : r_cond; -  array buffer=allocate(8); -  int r_ptr, w_ptr; -  -  int size() -  { -  return w_ptr - r_ptr; -  } -  -  mixed read() -  { -  while(!(w_ptr - r_ptr)) { -  // Make a MutexKey for wait() to please 7.3. This will of course -  // not fix the race, but we ignore that. See the discussion -  // above. (Must have an extra ref to the mutex since the -  // MutexKey doesn't keep one.) -  Thread.Mutex m = Thread.Mutex(); -  r_cond::wait (m->lock()); -  } -  mixed tmp = buffer[r_ptr]; -  buffer[r_ptr++] = 0; // Throw away any references. -  return tmp; -  } -  -  mixed try_read() -  { -  if (!(w_ptr - r_ptr)) return ([])[0]; -  mixed tmp = buffer[r_ptr]; -  buffer[r_ptr++] = 0; // Throw away any references. -  return tmp; -  } -  -  // Warning: This function isn't thread safe. -  void write(mixed v) -  { -  if(w_ptr >= sizeof(buffer)) -  { -  buffer=buffer[r_ptr..]+allocate(8); -  w_ptr-=r_ptr; -  r_ptr=0; -  } -  buffer[w_ptr++]=v; -  r_cond::signal(); -  } - } - #endif -  - #ifndef NO_SLOW_REQ_BT - // This is a system to dump all threads whenever a request takes - // longer than a configurable timeout. -  - protected Pike.Backend slow_req_monitor; // Set iff slow req bt is enabled. - protected float slow_req_timeout, slow_be_timeout; -  - protected void slow_req_monitor_thread (Pike.Backend my_monitor) - { -  // my_monitor is just a safeguard to ensure we don't get multiple -  // monitor threads. -  Roxen.name_thread(this_thread(), "Slow Request Monitor"); -  while (slow_req_monitor == my_monitor) -  slow_req_monitor (3600.0); -  Roxen.name_thread(this_thread(), 0); - } -  - protected mixed slow_be_call_out; -  - protected void slow_be_before_cb() - { - #ifdef DEBUG -  if (this_thread() != backend_thread) error ("Run from wrong thread.\n"); - #endif -  if (Pike.Backend monitor = slow_be_call_out && slow_req_monitor) { -  monitor->remove_call_out (slow_be_call_out); -  slow_be_call_out = 0; -  -  float backend_rtime = -  (gethrtime() - thread_task_start_times[this_thread()]) / 1E6; -  thread_task_start_times[this_thread()] = 0; -  -  if (backend_rtime > slow_be_timeout) { -  report_slow_thread_finished (this_thread(), backend_rtime); -  } -  } - } -  - protected void slow_be_after_cb() - { -  // FIXME: This should try to compensate for delays due to the pike -  // gc, because it just causes noise here. - #ifdef DEBUG -  if (this_thread() != backend_thread) error ("Run from wrong thread.\n"); - #endif -  if (slow_be_timeout > 0.0) { -  if (Pike.Backend monitor = slow_req_monitor) { -  thread_task_start_times[this_thread()] = gethrtime(); -  slow_be_call_out = monitor->call_out (dump_slow_req, slow_be_timeout, -  this_thread(), slow_be_timeout); -  } -  } - } -  - void slow_req_count_changed() - { -  Pike.Backend monitor = slow_req_monitor; -  int count = query ("slow_req_bt_count"); -  -  if (count && monitor) { -  // Just a change of the count - nothing to do. -  } -  -  else if (count) { // Start. -  monitor = slow_req_monitor = Pike.SmallBackend(); -  Thread.thread_create (slow_req_monitor_thread, monitor); -  monitor->call_out (lambda () {}, 0); // Safeguard if there's a race. -  slow_be_timeout_changed(); -  } -  -  else if (monitor) { // Stop. -  slow_req_monitor = 0; -  monitor->call_out (lambda () {}, 0); // To wake up the thread. -  slow_be_timeout_changed(); -  } - } -  - void slow_req_timeout_changed() - { - #ifdef DEBUG -  if (query ("slow_req_bt_timeout") < 0) error ("Invalid timeout.\n"); - #endif -  slow_req_timeout = query ("slow_req_bt_timeout"); - } -  - void slow_be_timeout_changed() - { - #ifdef DEBUG -  if (query ("slow_be_bt_timeout") < 0) error ("Invalid timeout.\n"); - #endif -  slow_be_timeout = query ("slow_be_bt_timeout"); -  - #ifdef DEBUG -  if ((Pike.DefaultBackend->before_callback && -  Pike.DefaultBackend->before_callback != slow_be_before_cb) || -  (Pike.DefaultBackend->after_callback && -  Pike.DefaultBackend->after_callback != slow_be_after_cb)) -  werror ("Pike.DefaultBackend already hooked up with " -  "other before/after callbacks - they get overwritten: %O/%O\n", -  Pike.DefaultBackend->before_callback, -  Pike.DefaultBackend->after_callback); - #endif -  -  if (query ("slow_req_bt_count") && slow_be_timeout > 0.0 && -  // Don't trig if we're shutting down. -  !shutdown_started) { -  Pike.DefaultBackend->before_callback = slow_be_before_cb; -  Pike.DefaultBackend->after_callback = slow_be_after_cb; -  } -  else { -  Pike.DefaultBackend->before_callback = 0; -  Pike.DefaultBackend->after_callback = 0; -  if (Pike.Backend monitor = slow_be_call_out && slow_req_monitor) { -  monitor->remove_call_out (slow_be_call_out); -  slow_be_call_out = 0; -  } -  } - } -  - protected int last_dump_hrtime; -  - protected void dump_slow_req (Thread.Thread thread, float timeout) - { -  object threads_disabled = _disable_threads(); -  -  int count = query ("slow_req_bt_count"); -  if (count > 0) set ("slow_req_bt_count", count - 1); -  -  if (thread == backend_thread && !slow_be_call_out) { -  // Avoid false alarms for the backend thread if we got here due to -  // a race. Should perhaps have something like this for the handler -  // threads too, but otoh races are more rare there due to the -  // longer timeouts. -  } -  -  else { -  string th_name = -  ((thread != backend_thread) && Roxen.thread_name(thread, 1)) || ""; -  if (sizeof(th_name)) -  th_name = " - " + th_name + " -"; -  report_debug ("###### %s 0x%x%s has been busy for more than %g seconds.\n", -  thread == backend_thread ? "Backend thread" : "Thread", -  thread->id_number(), th_name, timeout); -  int hrnow = gethrtime(); -  if ((hrnow - last_dump_hrtime) / 1E6 < slow_req_timeout / 2) { -  describe_thread (thread); -  } else { -  last_dump_hrtime = hrnow; -  mixed err = catch { -  describe_all_threads(0, 1); -  }; -  if (err) master()->handle_error(err); -  } -  } -  -  threads_disabled = 0; // Paranoia. - } -  - protected void report_slow_thread_finished (Thread.Thread thread, -  float time_spent) - { -  if (query ("slow_req_bt_count") == 0) { -  return; -  } -  -  string th_name = -  ((thread != backend_thread) && Roxen.thread_name(thread, 1)) || ""; -  if (sizeof(th_name)) -  th_name = " - " + th_name + " -"; -  -  report_debug ("###### %s 0x%x%s finished after %.2f seconds.\n", -  (thread == backend_thread ? -  "Backend thread" : "Thread"), -  thread->id_number(), th_name, time_spent); - } -  - #endif // !NO_SLOW_REQ_BT -  - // // This is easier than when there are no threads. - // // See the discussion below. :-) - // - // // But there is extra functionality below we really want, though, - // // so let's use that one instead... - // function async_sig_start( function f, int really ) - // { - // return lambda( mixed ... args ) { - // thread_create( f, @args ); - // }; - // } - local protected Queue low_handle_queue = Queue(); - local protected Queue handle_queue = Queue(); - //! Queues of things to handle. - //! - //! An entry consists of an @expr{array(function fp, array args)@}. - //! - //! @[low_handle_queue] is the queue that is used until all - //! configurations have loaded, when @[handle_queue] starts - //! getting used. - //! - //! Any entries in the @[handle_queue] are then transferred - //! to the @[low_handle_queue] (to preserve priorities), - //! and they are set to the same @[Queue] object (ie the - //! one that started life as @[low_handle_queue]. -  - local protected int thread_reap_cnt; - //! Number of handler threads in the process of being stopped. -  - protected int threads_on_hold; - //! Number of handler threads on hold. -  - // Global variables for statistics - int handler_num_runs = 0; - int handler_num_runs_001s = 0; - int handler_num_runs_005s = 0; - int handler_num_runs_015s = 0; - int handler_num_runs_05s = 0; - int handler_num_runs_1s = 0; - int handler_num_runs_5s = 0; - int handler_num_runs_15s = 0; - int handler_acc_time = 0; - int handler_acc_cpu_time = 0; -  - protected string debug_format_queue_task (array(function|array) task) - // Debug formatter of an entry in the handler or background_run queues. - { -  return ((functionp (task[0]) ? -  sprintf ("%s: %s", Function.defined (task[0]), -  master()->describe_function (task[0])) : -  programp (task[0]) ? -  sprintf ("%s: %s", Program.defined (task[0]), -  master()->describe_program (task[0])) : -  sprintf ("%O", task[0])) + -  "(" + -  map (task[1], lambda (mixed arg) -  {return RXML.utils.format_short (arg, 200);}) * ", " + -  ")"); - } -  - protected mapping(Thread.Thread:int) thread_task_start_times = ([]); -  - mapping(Thread.Thread:int) get_thread_task_start_times() - { -  // Also needed in Admin interface's thread wizard -  return thread_task_start_times + ([ ]); - } -  - local protected void handler_thread(int id) - //! The actual handling function. This functions read function and - //! parameters from the queue, calls it, then reads another one. There - //! is a lot of error handling to ensure that nothing serious happens if - //! the handler function throws an error. - { -  THREAD_WERR("Handle thread ["+id+"] started"); -  mixed h, q; -  set_u_and_gid (1); - #ifdef TEST_EUID_CHANGE -  if (test_euid_change) { -  Stdio.File f = Stdio.File(); -  if (f->open ("rootonly", "r") && f->read()) -  werror ("Handler thread %d can read rootonly\n", id); -  else -  werror ("Handler thread %d can't read rootonly\n", id); -  } - #endif -  while(1) -  { -  int thread_flagged_as_busy; - #ifndef NO_SLOW_REQ_BT -  Pike.Backend monitor; -  mixed call_out; - #endif -  if(q=catch { -  do { - // if (!busy_threads) werror ("GC: %d\n", gc()); -  cache_clear_deltas(); -  THREAD_WERR("Handle thread ["+id+"] waiting for next event"); -  if(arrayp(h = low_handle_queue->read()) && h[0]) { -  THREAD_WERR(sprintf("Handle thread [%O] calling %s", -  id, debug_format_queue_task (h))); -  set_locale(); -  busy_threads++; -  thread_flagged_as_busy = 1; -  handler_num_runs++; -  -  int start_hrtime = gethrtime(); -  thread_task_start_times[this_thread()] = start_hrtime; -  -  float handler_vtime = gauge { - #ifndef NO_SLOW_REQ_BT -  if (h[0] != bg_process_queue && -  // Leave out bg_process_queue. It makes a timeout on -  // every individual job instead. -  (monitor = slow_req_monitor) && slow_req_timeout > 0.0) { -  call_out = monitor->call_out (dump_slow_req, slow_req_timeout, -  this_thread(), slow_req_timeout); -  h[0](@h[1]); -  monitor->remove_call_out (call_out); -  } -  else - #endif -  { -  h[0](@h[1]); -  } -  }; -  int end_hrtime = gethrtime(); -  -  float handler_rtime = (end_hrtime - start_hrtime)/1E6; -  thread_task_start_times[this_thread()] = 0; -  - #ifndef NO_SLOW_REQ_BT -  if (slow_req_timeout > 0.0 && -  handler_rtime > slow_req_timeout) { -  report_slow_thread_finished (this_thread(), handler_rtime); -  } - #endif -  -  h=0; -  busy_threads--; -  thread_flagged_as_busy = 0; -  if (handler_rtime > 0.01) handler_num_runs_001s++; -  if (handler_rtime > 0.05) handler_num_runs_005s++; -  if (handler_rtime > 0.15) handler_num_runs_015s++; -  if (handler_rtime > 0.50) handler_num_runs_05s++; -  if (handler_rtime > 1.00) handler_num_runs_1s++; -  if (handler_rtime > 5.00) handler_num_runs_5s++; -  if (handler_rtime > 15.00) handler_num_runs_15s++; -  handler_acc_cpu_time += (int)(1E6*handler_vtime); -  handler_acc_time += (int)(1E6*handler_rtime); -  } else if(!h) { -  // Roxen is shutting down. -  report_debug("Handle thread ["+id+"] stopped.\n"); -  thread_reap_cnt--; - #ifdef NSERIOUS -  if(!thread_reap_cnt) report_debug("+++ATH\n"); - #endif -  return; -  } - #ifdef DEBUG -  else if (h != 1) -  error ("Unknown message in handle_queue: %O\n", h); - #endif -  else { -  num_hold_messages--; -  THREAD_WERR("Handle thread [" + id + "] put on hold"); -  threads_on_hold++; -  if (Thread.Condition cond = hold_wakeup_cond) { -  // Make a MutexKey for wait() to please 7.3. This will of -  // course not fix the race, but we ignore that. See the -  // comment at the declaration of hold_wakeup_cond. (Must -  // have an extra ref to the mutex since the MutexKey -  // doesn't keep one.) -  Thread.Mutex m = Thread.Mutex(); -  cond->wait (m->lock()); -  } -  threads_on_hold--; -  THREAD_WERR("Handle thread [" + id + "] released"); -  } -  } while(1); -  }) { -  if (thread_flagged_as_busy) -  busy_threads--; - #ifndef NO_SLOW_REQ_BT -  if (call_out) monitor->remove_call_out (call_out); - #endif -  if (h = catch { -  master()->handle_error (q); -  if (q = catch {h = 0;}) { -  report_error(LOC_M(5, "Uncaught error in handler thread: %sClient " -  "will not get any response from Roxen.")+"\n", -  describe_backtrace(q)); -  catch (q = 0); -  } -  }) { -  catch { -  report_error("Error reporting error:\n"); -  report_error(sprintf("Raw error: %O\n", h[0])); -  report_error(sprintf("Original raw error: %O\n", q[0])); -  }; -  catch (q = 0); -  catch (h = 0); -  } -  } -  } - } -  - //! Like @[handle()], but available before all configurations - //! have been loaded. - void low_handle(function f, mixed ... args) - { -  low_handle_queue->write(({f, args })); - } -  - void handle(function f, mixed ... args) - { -  handle_queue->write(({f, args })); - } -  - int handle_queue_length() - { -  return ((handle_queue != low_handle_queue) && low_handle_queue->size()) + -  handle_queue->size(); - } -  - int number_of_threads; - //! The number of handler threads to run. -  - int busy_threads; - //! The number of currently busy threads. -  - protected array(object) handler_threads = ({}); - //! The handler threads, the list is kept for debug reasons. -  - protected void start_low_handler_threads() - { -  if (query("numthreads") <= 1) { -  set( "numthreads", 1 ); -  report_warning (LOC_S(1, "Starting one thread to handle requests.")+"\n"); -  } else { -  report_notice (LOC_S(2, "Starting %d threads to handle requests.")+"\n", -  query("numthreads") ); -  } -  array(object) new_threads = ({}); -  for(; number_of_threads < query("numthreads"); number_of_threads++) -  new_threads += ({ do_thread_create( "Handle Thread [" + -  number_of_threads + "]", -  handler_thread, number_of_threads ) }); -  handler_threads += new_threads; - } -  - protected void transfer_handler_queue(Queue from, Queue to, int|void count) - { -  int transferred = handle_queue->size(); -  -  while (array entry = handle_queue->try_read()) { -  if (arrayp(entry)) { -  low_handle_queue->write(entry); -  } -  } -  -  // Some paranoia with respect to racing handle() vs start_handler_threads(). -  if (!transferred) { -  if (++count > 2) return; -  } -  -  low_handle(transfer_handler_queue, from, to, count); - } -  - void start_handler_threads() - { -  Queue handle_queue = this_program::handle_queue; -  this_program::handle_queue = low_handle_queue; -  -  transfer_handler_queue(handle_queue, low_handle_queue); - } -  - protected int num_hold_messages; - protected Thread.Condition hold_wakeup_cond = Thread.Condition(); - // Note: There are races in the use of this condition variable, but - // the only effect of that is that some handler thread might be - // considered hung when it's actually waiting on hold_wakeup_cond, and - // the hold/release handler threads function deal with hung threads - // anyway. The outcome would only be that release_handler_threads - // starts some extra handler thread unnecessarily. -  - void hold_handler_threads() - //! Tries to put all handler threads on hold, but gives up if it takes - //! too long. - { -  if (!hold_wakeup_cond) { -  THREAD_WERR("Ignoring request to hold handler threads during stop"); -  return; -  } -  -  int timeout=10; - #if constant(_reset_dmalloc) -  // DMALLOC slows stuff down a bit... -  timeout *= 10; - #endif /* constant(_reset_dmalloc) */ -  -  THREAD_WERR("Putting " + (number_of_threads - threads_on_hold) + -  " handler threads on hold, " + -  threads_on_hold + " on hold already"); -  -  for (int i = number_of_threads - threads_on_hold - num_hold_messages; i-- > 0;) { -  handle_queue->write (1); -  num_hold_messages++; -  } -  while (threads_on_hold < number_of_threads && timeout--) -  sleep (0.1); -  -  THREAD_WERR(threads_on_hold + " handler threads on hold, " + -  (number_of_threads - threads_on_hold) + " not responding"); - } -  - void release_handler_threads (int numthreads) - //! Releases any handler threads put on hold. If necessary new threads - //! are started to ensure that at least @[numthreads] threads are - //! responding. Threads that haven't arrived to the hold state since - //! @[hold_handler_threads] are considered nonresponding. - { -  if (Thread.Condition cond = hold_wakeup_cond) { -  // Flush out any remaining hold messages from the queue. -  for (int i = handle_queue->size(); i && num_hold_messages; i--) { -  mixed task = handle_queue->try_read(); -  if (task == 1) num_hold_messages--; -  else handle_queue->write (task); -  } - #ifdef DEBUG -  if (num_hold_messages) -  error ("num_hold_messages is bogus (%d).\n", num_hold_messages); - #endif -  num_hold_messages = 0; -  -  int blocked_threads = number_of_threads - threads_on_hold; -  int threads_to_create = numthreads - threads_on_hold; -  -  THREAD_WERR("Releasing " + threads_on_hold + " threads on hold"); -  cond->broadcast(); -  -  if (threads_to_create > 0) { -  array(object) new_threads = ({}); -  for (int n = 0; n < threads_to_create; number_of_threads++, n++) -  new_threads += ({ do_thread_create( "Handle Thread [" + -  number_of_threads + "]", -  handler_thread, number_of_threads ) }); -  handler_threads += new_threads; -  report_notice ("Created %d new handler threads to compensate " -  "for %d blocked ones.\n", threads_to_create, blocked_threads); -  } -  } -  else { -  THREAD_WERR("Ignoring request to release handler threads during stop"); -  return; -  } - } -  - //! - int handler_threads_on_hold() {return !!hold_wakeup_cond;} -  - protected Thread.MutexKey backend_block_lock; -  - void stop_handler_threads() - //! Stop all the handler threads and the backend, but give up if it - //! takes too long. - { -  int timeout=15; // Timeout if the bg queue doesn't get shorter. -  int background_run_timeout = 100; // Hard timeout that cuts off the bg queue. - #if constant(_reset_dmalloc) -  // DMALLOC slows stuff down a bit... -  timeout *= 10; -  background_run_timeout *= 3; - #endif /* constant(_reset_dmalloc) */ -  -  report_debug("Stopping all request handler threads.\n"); -  -  // Wake up any handler threads on hold, and ensure none gets on hold -  // after this. -  if (Thread.Condition cond = hold_wakeup_cond) { -  hold_wakeup_cond = 0; -  cond->broadcast(); -  } -  -  while(number_of_threads>0) { -  number_of_threads--; -  low_handle_queue->write(0); -  thread_reap_cnt++; -  } -  handler_threads = ({}); -  -  if (this_thread() != backend_thread && !backend_block_lock) { -  thread_reap_cnt++; -  Thread.Mutex mutex = Thread.Mutex(); -  backend_block_lock = mutex->lock(); -  call_out (lambda () { -  thread_reap_cnt--; -  report_debug("Backend thread stopped.\n"); -  mutex->lock(); -  error("Backend stop failed.\n"); -  }, 0); -  } -  -  int prev_bg_len = bg_queue_length(); -  -  while (thread_reap_cnt) { -  sleep(0.1); -  -  if (--timeout <= 0) { -  int cur_bg_len = bg_queue_length(); -  if (prev_bg_len < cur_bg_len) -  // Allow more time if the background queue is being worked off. -  timeout = 10; -  prev_bg_len = cur_bg_len; -  } -  -  if(--background_run_timeout <= 0 || timeout <= 0) { -  report_debug("Giving up waiting on threads; " -  "%d threads blocked, %d jobs in the background queue.\n", -  thread_reap_cnt, bg_queue_length()); - #ifdef DEBUG -  describe_all_threads(); - #endif -  return; -  } -  } - } -  - #else - // handle function used when THREADS is not enabled. -  void handle(function f, mixed ... args) - { -  f(@args); - } -  - // function handle = unthreaded_handle; -  - #endif /* THREADS */ -  - function async_sig_start( function f, int really ) - { -  class SignalAsyncVerifier( function f ) -  { -  protected int async_called; -  -  void really_call( array args ) -  { -  async_called = 0; -  f( @args ); -  } -  -  void call( mixed ... args ) -  { -  if( async_called && async_called-time() ) -  { -  report_debug("Received signal %s\n", (string) signame( args[0] ) ); -  report_debug("\n\n" -  "Async calling failed for %O, calling synchronous\n", f); -  report_debug("Backtrace at time of hangup:\n%s\n", -  describe_backtrace( backtrace() )); -  f( @args ); -  return; -  } -  if( !async_called ) // Do not queue more than one call at a time. -  { -  report_debug("Received signal %s\n", (string) signame( args[0] ) ); -  async_called=time(); -  call_out( really_call, 0, args ); -  } -  } -  }; -  // call_out is not really useful here, since we probably want to run -  // the signal handler immediately, not whenever the backend thread -  // is available. /per -  // -  // Calling directly like this may however lead to recursive mutex -  // lock errors. The problem cannot be solved using lock(2) since the -  // internal structures may be in an inconsistent state from the -  // previous call, and waiting for the lock probably leads to a -  // deadlock. /noring -  // -  // But on the other hand, you are not very likely to have any mutex -  // locks in an unthreaded pike, since it's quite impossible. /per -  // -  // But still, the problems with inconsistent internal states are -  // there. The API:s for many (thread safe) objects are designed to -  // only allow one (1) caller at any given time. It's a bug if this -  // restriction can be circumvented using signals. I suggest that -  // Thread.Mutex takes care of this problem in non-threaded mode. -  // / noring -  // -  // Apparantly it already did that. :-) -  // -  // I also fixed SIGHUP to be somewhat more asynchronous. -  // -  // I also added a rather small amount of magic so that it is called -  // asynchronously the first time it is received, but repeated -  // signals are not called asynchronously unless the first signal -  // handler was actually called. -  // -  // Except for the SIGQUIT signal, which dumps a backtrace. It would -  // be an excercise in futility to call that one asynchronously. -  // -  // I hope that this will solve all your problems. /per -  if( really > 0 ) -  return lambda( mixed ... args ){ call_out( f, 0, @args ); }; -  if( really < 0 ) -  return f; -  return SignalAsyncVerifier( f )->call; - } -  - #ifdef THREADS - protected Thread.Queue bg_queue = Thread.Queue(); - protected Thread.Thread bg_process_thread; -  - // Use a time buffer to strike a balance if the server is busy and - // always have at least one busy thread: The maximum waiting time in - // that case is somewhere between bg_time_buffer_min and - // bg_time_buffer_max. If there are only short periods of time between - // the queue runs, the max waiting time will shrink towards the - // minimum. - protected constant bg_time_buffer_max = 30; - protected constant bg_time_buffer_min = 0; - protected int bg_last_busy = 0; - int bg_num_runs = 0; - int bg_num_runs_001s = 0; - int bg_num_runs_005s = 0; - int bg_num_runs_015s = 0; - int bg_num_runs_05s = 0; - int bg_num_runs_1s = 0; - int bg_num_runs_5s = 0; - int bg_num_runs_15s = 0; - int bg_acc_time = 0; - int bg_acc_cpu_time = 0; -  - int bg_queue_length() - { -  return bg_queue->size(); - } -  - protected void bg_process_queue() - { -  if (bg_process_thread) return; -  // Relying on the interpreter lock here. -  bg_process_thread = this_thread(); -  -  int maxbeats = -  min (time() - bg_last_busy, bg_time_buffer_max) * (int) (1 / 0.01); -  - #ifndef NO_SLOW_REQ_BT -  Pike.Backend monitor; -  mixed call_out; - #endif -  -  if (mixed err = catch { -  // Make sure we don't run forever if background jobs are queued -  // over and over again, to avoid starving other threads. (If they -  // are starved enough, busy_threads will never be incremented.) -  // If jobs are enqueued while running another background job, -  // bg_process_queue is put on the handler queue again at the very -  // end of this function. -  // -  // However, during shutdown we continue until the queue is really -  // empty. background_run won't queue new jobs then, and if this -  // takes too long, stop_handler_threads will exit anyway. -  int jobs_to_process = bg_queue->size(); -  while (hold_wakeup_cond ? jobs_to_process-- : bg_queue->size()) { -  // Not a race here since only one thread is reading the queue. -  array task = bg_queue->read(); -  -  // Wait a while if another thread is busy already. -  if (busy_threads > 1) { -  for (maxbeats = max (maxbeats, (int) (bg_time_buffer_min / 0.01)); -  busy_threads > 1 && maxbeats > 0; -  maxbeats--) -  sleep (0.01); -  bg_last_busy = time(); -  } -  - #ifdef DEBUG_BACKGROUND_RUN -  report_debug ("background_run run %s [%d jobs left in queue]\n", -  debug_format_queue_task (task), -  bg_queue->size()); - #endif -  -  float task_vtime, task_rtime; -  bg_num_runs++; -  - #ifndef NO_SLOW_REQ_BT -  if ((monitor = slow_req_monitor) && slow_req_timeout > 0.0) { -  call_out = monitor->call_out (dump_slow_req, slow_req_timeout, -  this_thread(), slow_req_timeout); -  int start_hrtime = gethrtime(); -  thread_task_start_times[this_thread()] = start_hrtime; -  task_vtime = gauge { -  if (task[0]) // Ignore things that have become destructed. -  // Note: BackgroundProcess.repeat assumes that there are -  // exactly two refs to task[0] during the call below. -  task[0] (@task[1]); -  }; -  task_rtime = (gethrtime() - start_hrtime) / 1e6; -  thread_task_start_times[this_thread()] = 0; -  monitor->remove_call_out (call_out); -  } -  else - #endif -  { -  int start_hrtime = gethrtime(); -  thread_task_start_times[this_thread()] = start_hrtime; -  task_vtime = gauge { -  if (task[0]) -  task[0] (@task[1]); -  }; -  task_rtime = (gethrtime() - start_hrtime) / 1e6; -  thread_task_start_times[this_thread()] = 0; -  } -  -  if (task_rtime > 0.01) bg_num_runs_001s++; -  if (task_rtime > 0.05) bg_num_runs_005s++; -  if (task_rtime > 0.15) bg_num_runs_015s++; -  if (task_rtime > 0.50) bg_num_runs_05s++; -  if (task_rtime > 1.00) bg_num_runs_1s++; -  if (task_rtime > 5.00) bg_num_runs_5s++; -  if (task_rtime > 15.00) bg_num_runs_15s++; -  bg_acc_cpu_time += (int)(1E6*task_vtime); -  bg_acc_time += (int)(1E6*task_rtime); -  -  if (task_rtime > 60.0) -  report_warning ("Warning: Background job took more than one minute " -  "(%g s real time and %g s cpu time):\n" -  " %s (%s)\n%s", -  task_rtime, task_vtime, -  functionp (task[0]) ? -  sprintf ("%s: %s", Function.defined (task[0]), -  master()->describe_function (task[0])) : -  programp (task[0]) ? -  sprintf ("%s: %s", Program.defined (task[0]), -  master()->describe_program (task[0])) : -  sprintf ("%O", task[0]), -  map (task[1], lambda (mixed arg) -  {return sprintf ("%O", arg);}) * ", ", -  bg_queue->size() ? -  (bg_queue->size() > 1 ? -  " " + bg_queue->size() + " more jobs in the " -  "background queue were delayed.\n" : -  " 1 more job in the background queue was delayed.\n"): -  ""); - #ifdef DEBUG_BACKGROUND_RUN -  else -  report_debug ("background_run done, " -  "took %g ms cpu time and %g ms real time\n", -  task_vtime * 1000, task_rtime * 1000); - #endif -  -  if (busy_threads > 1) bg_last_busy = time(); -  } -  }) { - #ifndef NO_SLOW_REQ_BT -  if (call_out) monitor->remove_call_out (call_out); - #endif -  bg_process_thread = 0; -  handle (bg_process_queue); -  throw (err); -  } -  bg_process_thread = 0; -  if (bg_queue->size()) { -  handle (bg_process_queue); -  // Sleep a short while to encourage a thread switch. This is a -  // kludge to avoid starving non-handler threads, since pike -  // currently (7.8.503) doesn't switch threads reliably on yields. -  sleep (0.001); -  } - } - #endif -  - Thread.Thread background_run_thread() - //! Returns the thread currently executing the background_run queue, - //! or 0 if it isn't being processed. - { - #ifdef THREADS -  return bg_process_thread; - #else -  // FIXME: This is not correct. Should return something nonzero when -  // called from the call out. But noone is running without threads -  // nowadays anyway. -  return 0; - #endif - } -  - mixed background_run (int|float delay, function func, mixed... args) - //! Enqueue a task to run in the background in a way that makes as - //! little impact as possible on the incoming requests. No matter how - //! many tasks are queued to run in the background, only one is run at - //! a time. The tasks won't be starved regardless of server load, - //! though. - //! - //! The function @[func] will be enqueued after approximately @[delay] - //! seconds, to be called with the rest of the arguments as its - //! arguments. - //! - //! In a multithreaded server the function will be executed in one of - //! the handler threads. The function is executed in the backend - //! thread if no thread support is available, although that - //! practically never occurs anymore. - //! - //! To avoid starving other background jobs, the function should never - //! run for a long time. Instead do another call to @[background_run] - //! to queue it up again after some work has been done, or use - //! @[BackgroundProcess]. - //! - //! @returns - //! If the function is queued for execution right away then zero is - //! returned. Otherwise its call out identifier is returned, which can - //! be used with @[find_call_out] or @[remove_call_out]. - { -  // FIXME: Make it possible to associate the background job with a -  // RoxenModule or Configuration, so that report_error etc can log in -  // a good place. - #ifdef DEBUG_BACKGROUND_RUN -  report_debug ("background_run enqueue %s (%s) [%d jobs in queue]\n", -  functionp (func) ? -  sprintf ("%s: %s", Function.defined (func), -  master()->describe_function (func)) : -  programp (func) ? -  sprintf ("%s: %s", Program.defined (func), -  master()->describe_program (func)) : -  sprintf ("%O", func), -  map (args, lambda (mixed arg) -  {return sprintf ("%O", arg);}) * ", ", -  bg_queue->size()); - #endif -  - #ifdef THREADS -  if (!hold_wakeup_cond) { -  // stop_handler_threads is running; ignore more work. - #ifdef DEBUG -  report_debug ("Ignoring background job queued during shutdown: %O\n", func); - #endif -  return 0; -  } -  -  class enqueue(function func, mixed ... args) -  { -  int __hash() { return hash_value(func); } -  int `==(mixed gunc) { return func == gunc; } -  string _sprintf() { return sprintf("background_run(%O)", func); } -  mixed `()() -  { -  bg_queue->write (({func, args})); -  if (!bg_process_thread) -  handle (bg_process_queue); -  } -  }; -  -  if (delay) -  return call_out (enqueue(func, @args), delay); -  else { -  enqueue(func, @args)(); -  return 0; -  } - #else -  // Can't do much better when we haven't got threads.. -  return call_out (func, delay, @args); - #endif - } -  - class BackgroundProcess - //! A class to do a task repeatedly in the background, in a way that - //! makes as little impact as possible on the incoming requests (using - //! @[background_run]). - //! - //! The user must keep a reference to this object, otherwise it will remove - //! itself and the callback won't be called anymore. - { -  int|float period; -  function func; -  array args; -  int stopping = 0; -  -  protected void repeat (function func, array args) -  { -  // Got a minimum of four refs to this: -  // o One in the task array in bg_process_queue. -  // o One on the stack in the call in bg_process_queue. -  // o One as current_object in the stack frame. -  // o One on the stack as argument to Debug.refs. -  int self_refs = Debug.refs (this); - #ifdef DEBUG -  if (self_refs < 4) -  error ("Minimum ref calculation wrong - have only %d refs.\n", self_refs); - #endif -  if (stopping || (self_refs <= 4) || !func) { -  stopping = 2; // Stopped. -  return; -  } -  mixed err = catch { -  func (@args); -  }; -  if (err) -  master()->handle_error (err); -  background_run (period, repeat, func, args); -  } -  -  //! @decl void set_period (int|float period); -  //! -  //! Changes the period to @[period] seconds between calls. -  //! -  //! @note -  //! This does not change the currently ongoing period, if any. That -  //! might be remedied. -  void set_period (int|float period_) -  { -  period = period_; -  } -  -  //! @decl protected void create (int|float period, function func, mixed... args); -  //! -  //! The function @[func] will be called with the following arguments -  //! after approximately @[period] seconds, and then kept being -  //! called with approximately that amount of time between each call. -  //! -  //! The repetition will stop if @[stop] is called, or if @[func] -  //! throws an error. -  protected void create (int|float period_, function func_, mixed... args_) -  { -  period = period_; -  func = func_; -  args = args_; -  background_run (period, repeat, func, args); -  } -  -  void stop() -  //! Sets a flag to stop the succession of calls. -  { -  stopping |= 1; -  } -  -  void start() -  //! Restart a stopped process. -  { -  int state = stopping; -  stopping = 0; -  if (state & 2) { -  background_run (period, repeat, func, args); -  } -  } -  -  string _sprintf() -  { -  return sprintf("BackgroundProcess(%O, %O)", period, func); -  } - } -  -  - mapping get_port_options( string key ) - //! Get the options for the key 'key'. - //! The interpretation of the options is protocol specific. - { -  return (query( "port_options" )[ key ] || ([])); - } -  - void set_port_options( string key, mapping value ) - //! Set the options for the key 'key'. - //! The interpretation of the options is protocol specific. - { -  mapping q = query("port_options"); -  q[ key ] = value; -  set( "port_options" , q ); -  save( ); - } -  - #ifdef DEBUG_URL2CONF - #define URL2CONF_MSG(X...) report_debug (X) - #else - #define URL2CONF_MSG(X...) - #endif -  - protected mapping(string:int(0..1)) host_is_local_cache = ([]); -  - //! Check if @[hostname] is local to this machine. - int(0..1) host_is_local(string hostname) - { -  int(0..1) res; -  if (!zero_type(res = host_is_local_cache[hostname])) return res; -  -  // Look up the IP. -  string ip = blocking_host_to_ip(hostname); -  -  // Can we bind to it? -  Stdio.Port port = Stdio.Port(); -  // bind() can trow error if ip is an invalid hostname. -  catch { -  res = port->bind(0, 0, ip); -  }; -  -  destruct(port); -  return host_is_local_cache[hostname] = res; - } -  - array(Protocol|mapping(string:mixed)) find_port_for_url ( -  Standards.URI url, void|Configuration only_this_conf) - // Returns ({port_obj, url_data}) for a url that matches the given - // one. url_data is the mapping for the url in port_obj->urls. If - // only_this_conf is given then only ports for that configuration are - // searched. - { -  // Cannot use the uri formatter in Standards.URI here since we -  // always want the port number to be present. -  string host = url->host; -  if (has_value (host, ":")) -  host = "[" + (Protocols.IPv6.normalize_addr_basic (host) || host) + "]"; -  string url_with_port = sprintf ("%s://%s:%d%s", url->scheme, host, url->port, -  sizeof (url->path) ? url->path : "/"); -  -  URL2CONF_MSG("URL with port: %s\n", url_with_port); -  -  foreach (urls; string u; mapping(string:mixed) q) -  { -  URL2CONF_MSG("Trying %O:%O\n", u, q); -  if( glob( u+"*", url_with_port ) ) -  { -  URL2CONF_MSG("glob match\n"); -  if (Protocol p = q->port) -  if (mapping(string:mixed) url_data = -  p->find_url_data_for_url (url_with_port, 0, 0)) -  { -  Configuration c = url_data->conf; -  URL2CONF_MSG("Found config: %O\n", url_data->conf); -  -  if ((only_this_conf && (c != only_this_conf)) || -  (sscanf (u, "%*s://%*[^*?]%*c") == 3 && // u contains * or ?. -  // u is something like "http://*:80/" -  (!host_is_local(url->host)))) { -  // Bad match. -  URL2CONF_MSG("Bad match: only_this_conf:%O, host_is_local:%O\n", -  (only_this_conf && (c == only_this_conf)), -  (!host_is_local(url->host))); -  c = 0; -  continue; -  } -  -  URL2CONF_MSG("Result: %O\n", c); -  return ({p, url_data}); -  } -  } -  } -  -  return ({0, 0}); - } -  - Configuration find_configuration_for_url(Standards.URI url, -  void|Configuration only_this_conf, -  void|array(Protocol) return_port) - //! Tries to to determine if a request for the given url would end up - //! in this server, and if so returns the corresponding configuration. - //! - //! If @[only_this_conf] has been specified only matches against it - //! will be returned. - { -  [Protocol port_obj, mapping(string:mixed) url_data] = -  find_port_for_url (url, only_this_conf); -  if (return_port) -  return_port[0] = port_obj; -  return url_data && url_data->conf; - } -  - class InternalRequestID - //! ID for internal requests that are not linked to any real request. - { -  inherit RequestID; -  -  this_program set_path( string f ) -  { -  raw_url = Roxen.http_encode_invalids( f ); -  -  if( strlen( f ) > 5 ) -  { -  string a; -  switch( f[1] ) -  { -  case '<': -  if (sscanf(f, "/<%s>/%s", a, f)==2) -  { -  config = (multiset)(a/","); -  f = "/"+f; -  } -  // intentional fall-through -  case '(': -  if(strlen(f) && sscanf(f, "/(%s)/%s", a, f)==2) -  { -  prestate = (multiset)( a/","-({""}) ); -  f = "/"+f; -  } -  } -  } -  not_query = Roxen.simplify_path( scan_for_query( f ) ); -  return this; -  } -  -  this_program set_url( string url ) -  { -  object uri = Standards.URI(url); -  prot = upper_case(uri->scheme); -  misc->host = uri->host; -  if ((prot == "HTTP" && uri->port != 80) || -  (prot == "HTTPS" && uri->port != 443)) -  misc->host += ":" + uri->port; -  string path = uri->path; -  raw_url = path; -  method = "GET"; -  raw = "GET " + raw_url + " HTTP/1.1\r\n\r\n"; -  [port_obj, mapping(string:mixed) url_data] = find_port_for_url (uri, 0); -  if (url_data) { -  conf = url_data->conf; -  if (!conf->inited) conf->enable_all_modules(); -  if (string config_path = url_data->path) -  adjust_for_config_path (config_path); -  } -  -  // Update the cached URL base to keep url_base() happy. -  uri->path = (misc->site_prefix_path || "") + "/"; -  uri->query = UNDEFINED; -  uri->fragment = UNDEFINED; -  cached_url_base = sprintf("%s", uri); -  return set_path( raw_url ); -  } -  -  protected string _sprintf() -  { -  return sprintf("InternalRequestID(conf=%O; not_query=%O)", conf, not_query ); -  } -  -  protected void create() -  { -  client = ({ "Roxen" }); -  prot = "INTERNAL"; -  port_obj = InternalProtocol(); -  method = "GET"; -  real_variables = ([]); -  variables = FakedVariables( real_variables ); -  root_id = this_object(); -  cached_url_base = "internal://0.0.0.0:0/"; -  -  misc = ([ "pref_languages": PrefLanguages(), -  "cacheable": INITIAL_CACHEABLE, -  ]); -  connection_misc = ([]); -  cookies = CookieJar(); -  throttle = ([]); -  client_var = ([]); -  request_headers = ([]); -  prestate = (<>); -  config = (<>); -  supports = (<>); -  pragma = (<>); -  rest_query = ""; -  extra_extension = ""; -  remoteaddr = "127.0.0.1"; -  } - } -  - class Protocol - //! The basic protocol. - //! Implements reference handling, finding Configuration objects - //! for URLs, and the bind/accept handling. - { -  protected Stdio.Port port_obj; -  -  inherit "basic_defvar"; -  -  int bound; -  //! 0 if the port isn't bound, 1 if it is, and -1 if it binding it -  //! failed with EADDRINUSE when told to ignore that error. -  //! -  //! @note -  //! The -1 state should be uncommon since @[register_url] should -  //! remove such objects after the failed bind attempt. -  -  string path; -  constant name = "unknown"; -  -  constant supports_ipless = 0; -  //! If true, the protocol handles ip-less virtual hosting -  -  constant requesthandlerfile = ""; -  //! Filename of a by-connection handling class. It is also possible -  //! to set the 'requesthandler' class member in a overloaded create -  //! function. -  -  constant default_port = 4711; -  //! If no port is specified in the URL, use this one -  -  string url_prefix = name + "://"; -  -  int port; -  //! The currently bound portnumber -  -  string ip; -  //! The canonical IP-number (0 for ANY) this port is bound to. -  //! -  //! IPv6 numbers are in colon separated four-digit lower-case hexadecimal -  //! notation with the first longest sequence of zeros compressed. -  -  int refs; -  //! The number of references to this port. This is the same as the -  //! size of @[urls] and @[sorted_urls]. -  -  program requesthandler; -  //! The per-connection request handling class -  -  array(string) sorted_urls = ({}); -  //! Sorted by length, longest first -  -  mapping(string:mapping(string:mixed)) urls = ([]); -  //! .. url -> ([ "conf":.., ... ]) -  //! -  //! Indexed by URL. The following data is stored: -  //! @mapping -  //! @member Configuration "conf" -  //! The Configuration object for this URL. -  //! @member string "hostname" -  //! The hostname from the URL. -  //! @member string|void "path" -  //! The path (if any) from the URL. -  //! @member Protocol "port" -  //! The protocol handler for this URL. -  //! @member int "mib_version" -  //! (Only SNMP). The version number for the configuration MIB -  //! tree when it was last merged. -  //! @endmapping -  -  mapping(Configuration:mapping(string:mixed)) conf_data = ([]); -  //! Maps the configuration objects to the data mappings in @[urls]. -  -  void ref(string name, mapping(string:mixed) data) -  //! Add a ref for the URL @[name] with the data @[data]. -  //! -  //! See @[urls] for documentation about the supported -  //! fields in @[data]. -  { -  if(urls[name]) -  { -  conf_data[urls[name]->conf] = urls[name] = data; -  return; // only ref once per URL -  } -  if (!refs) path = data->path; -  else if (path != (data->path || "")) path = 0; -  refs++; -  mu = 0; -  conf_data[data->conf] = urls[name] = data; -  sorted_urls = Array.sort_array(indices(urls), -  lambda(string a, string b) { -  return sizeof(a)<sizeof(b); -  }); -  } -  -  void unref(string _name) -  //! Remove a ref for the URL '_name' -  { - // if(!urls[name]) // only unref once - // return; -  -  m_delete(conf_data, urls[_name]->conf); -  m_delete(urls, _name); -  if (!path) { -  array(string) paths = Array.uniq (values (urls)->path); -  if (sizeof (paths) == 1) path = paths[0]; -  } -  sorted_urls -= ({_name}); - #ifdef PORT_DEBUG -  report_debug("Protocol(%s)->unref(%O): refs:%d\n", -  get_url(), _name, refs); - #endif /* PORT_DEBUG */ -  if( !--refs ) { -  if (retries) { -  remove_call_out(bind); -  } -  if (port_obj) { -  destruct(port_obj); -  } -  port_obj = 0; -  if (open_ports[name]) { -  if (open_ports[name][ip]) { -  m_delete(open_ports[name][ip], port); -  if(!sizeof(open_ports[name][ip])) { -  // Make sure the entries for IPv4 and IPv6 ANY are left alone. -  if (ip && ip != "::") -  m_delete(open_ports[name], ip); -  } -  } -  if (sizeof(open_ports[name]) <= 2) { -  // Only ANY left. -  int empty = 1; -  foreach(open_ports[name]; string ip; mapping m) { -  if (sizeof(m)) { -  empty = 0; -  break; -  } -  } -  if (empty) -  m_delete(open_ports, name); -  } -  } -  any_port = 0; // Avoid possibly cyclic ref. -  //destruct( ); // Close the port. -  } -  } -  -  Stdio.File accept() -  { -  return port_obj->accept(); -  } -  -  string query_address() -  { -  return port_obj && port_obj->query_address(); -  } -  -  mapping(string:mixed) mu; -  string rrhf; -  protected void got_connection() -  { -  Stdio.File q; -  while( q = accept() ) -  { -  if( !requesthandler && rrhf ) -  { -  requesthandler = (program)(rrhf); -  } -  Configuration c; -  if( refs < 2 ) -  { -  if(!mu) -  { -  mu = get_iterator(urls)->value(); -  if(!(c=mu->conf)->inited ) { -  handle (lambda () { -  c->enable_all_modules(); -  call_out (lambda () -  { -  // The connection might have been -  // closed already, e.g. by -  // http_fallback.ssl_alert_callback() -  // (prot_https.pike). Avoid a -  // backtrace in that case. -  if (q->is_open()) -  requesthandler (q, this, c); -  }, 0); -  }); -  return; -  } -  } else -  c = mu->conf; -  } -  requesthandler( q, this_object(), c ); -  } -  } -  -  private Protocol any_port; -  -  mapping(string:mixed) find_url_data_for_url (string url, int no_default, -  RequestID id) -  { -  if( refs == 1 ) -  { -  if (!no_default) { -  if(!mu) mu=get_iterator(urls)->value(); -  URL2CONF_MSG ("%O %O Only one configuration: %O\n", -  this, url, mu->conf); -  return mu; -  } -  } else if (!refs) { -  URL2CONF_MSG("%O %O No active URLS!\n", this, url); -  return 0; -  } -  -  URL2CONF_MSG("sorted_urls: %O\n" -  "url: %O\n", sorted_urls, url); -  // The URLs are sorted from longest to shortest, so that short -  // urls (such as http://*/) will not match before more complete -  // ones (such as http://*.roxen.com/) -  foreach( sorted_urls, string in ) -  { -  if( glob( in+"*", url ) ) -  { -  URL2CONF_MSG ("%O %O sorted_urls: %O\n", this, url, urls[in]->conf); -  return urls[in]; -  } -  } -  -  if( no_default ) { -  URL2CONF_MSG ("%O %O no default\n", this, url); -  return 0; -  } -  -  // Note: The docs for RequestID.misc->default_conf has a -  // description of this fallback procedure. -  -  // No host matched, or no host header was included in the request. -  // Is the URL in the '*' ports? -  if (!any_port) -  any_port = open_ports[ name ][ 0 ][ port ]; -  if (any_port && any_port != this) -  if (mapping(string:mixed) u = -  any_port->find_url_data_for_url (url, 1, id)) { -  URL2CONF_MSG ("%O %O found on ANY port: %O\n", this, url, u->conf); -  if (id) { -  id->misc->defaulted_conf = 1; -  id->port_obj = any_port; -  } -  return u; -  } -  -  // No. We have to default to one of the other ports. -  // It might be that one of the servers is tagged as a default server. -  mapping(Configuration:int(1..1)) choices = ([]); -  foreach( configurations, Configuration c ) -  if( c->query( "default_server" ) ) -  choices[c] = 1; -  -  if( sizeof( choices ) ) -  { -  // Pick a default server bound to this port -  foreach (urls;; mapping cc) -  if( choices[ cc->conf ] ) -  { -  URL2CONF_MSG ("%O %O conf in choices: %O\n", this, url, cc->conf); -  if (id) id->misc->defaulted_conf = 2; -  return cc; -  } -  } -  -  return 0; -  } -  -  Configuration find_configuration_for_url( string url, RequestID id ) -  //! Given a url and requestid, try to locate a suitable configuration -  //! (virtual site) for the request. -  //! This interface is not at all set in stone, and might change at -  //! any time. -  { -  mapping(string:mixed) url_data = find_url_data_for_url (url, 0, id); -  -  if (!url_data) { -  // Pick the first default server available. FIXME: This makes -  // it impossible to handle the server path correctly. -  foreach (configurations, Configuration c) -  if (c->query ("default_server")) { -  URL2CONF_MSG ("%O %O any default server: %O\n", this, url, c); -  if (id) id->misc->defaulted_conf = 3; -  return c; -  } -  -  // if we end up here, there is no default port at all available -  // so grab the first configuration that is available at all. -  // We choose the last entry in sorted_urls since that's the most -  // generic one and therefore probably the best option for a -  // fallback. -  url_data = urls[sorted_urls[-1]]; -  if (id) { -  id->misc->defaulted_conf = 4; -  id->misc->defaulted=1; // Compat. -  } -  URL2CONF_MSG ("%O %O last in sorted_urls: %O\n", this, url, -  url_data->conf); -  } -  -  // It's assumed nothing below uses data in this object, since -  // find_url_data_for_url might have switched Protocol object. -  -  string config_path = url_data->path; -  if (config_path && id && id->adjust_for_config_path) -  id->adjust_for_config_path (config_path); -  Configuration c = url_data->conf; -  return c; -  } -  -  mixed query_option( string x ) -  //! Query the port-option 'x' for this port. -  { -  return query( x ); -  } -  -  string get_key() -  //! Return the key used for this port (protocol:ip:portno) -  { - #if 0 -  if (ip == "::") -  return name + ":0:" + port; -  else - #endif -  return name+":"+ip+":"+port; -  } -  -  string get_url() -  //! Return the port on URL form. -  { -  return (string) name + "://" + -  (!ip ? "*" : has_value (ip, ":") ? "[" + ip + "]" : ip) + -  ":" + port + "/"; -  } -  -  void save() -  //! Save all port options -  { -  set_port_options( get_key(), -  mkmapping( indices(variables), -  map(indices(variables),query))); -  } -  -  void restore() -  //! Restore all port options from saved values -  { -  foreach( (array)get_port_options( get_key() ), array kv ) -  set( kv[0], kv[1] ); -  } -  -  protected int retries; -  protected void bind (void|int ignore_eaddrinuse) -  { -  if (bound) return; -  if (!port_obj) port_obj = Stdio.Port(); -  Privs privs = Privs (sprintf ("Binding %s", get_url())); -  if (port_obj->bind(port, got_connection, ip)) -  { -  privs = 0; -  bound = 1; -  return; -  } -  privs = 0; - #if constant(System.EAFNOSUPPORT) -  if (port_obj->errno() == System.EAFNOSUPPORT) { -  // Fail permanently. -  error("Invalid address " + ip); -  } - #endif /* System.EAFNOSUPPORT */ - #if constant(System.EADDRINUSE) -  if (port_obj->errno() == System.EADDRINUSE) { -  if (ignore_eaddrinuse) { -  // Told to ignore the bind problem. -  bound = -1; -  return; -  } -  if (retries++ < 10) { -  // We may get spurious failures on rebinding ports on some OS'es -  // (eg Linux, WIN32). See [bug 3031]. -  report_error(LOC_M(6, "Failed to bind %s (%s)")+"\n", -  get_url(), strerror(port_obj->errno())); -  report_notice(LOC_M(62, "Attempt %d. Retrying in 1 minute.")+"\n", -  retries); -  call_out(bind, 60); -  } -  } -  else - #endif /* constant(System.EADDRINUSE) */ -  { -  report_error(LOC_M(6, "Failed to bind %s (%s)")+"\n", -  get_url(), strerror(port_obj->errno())); - #if 0 -  werror (describe_backtrace (backtrace())); - #endif -  } -  } -  -  string canonical_ip(string i) -  { -  if (!i) return 0; -  if (has_value(i, ":")) -  return Protocols.IPv6.normalize_addr_short (i); -  else { -  // IPv4 -  array(int) segments = array_sscanf(i, "%d.%d.%d.%d"); -  string bytes; -  switch(sizeof(segments)) { -  default: -  case 4: -  bytes = sprintf("%1c%1c%1c%1c", @segments); -  break; -  case 0: return 0; // ANY. -  case 1: -  /* When only one part is given, the value is stored directly in -  * the network address without any byte rearrangement. -  */ -  bytes = sprintf("%4c", @segments); -  break; -  case 2: -  /* When a two part address is supplied, the last part is inter- -  * preted as a 24-bit quantity and placed in the right most -  * three bytes of the network address. This makes the two part -  * address format convenient for specifying Class A network -  * addresses as net.host. -  */ -  bytes = sprintf("%1c%3c", @segments); -  break; -  case 3: -  /* When a three part address is specified, the last part is -  * interpreted as a 16-bit quantity and placed in the right -  * most two bytes of the network address. This makes the three -  * part address format convenient for specifying Class B net- -  * work addresses as 128.net.host. -  */ -  bytes = sprintf("%1c%1c%2c", @segments); -  break; -  } -  if (bytes == "\0\0\0\0") return 0; // ANY. -  return sprintf("%d.%d.%d.%d", @((array(int))bytes)); -  } -  } -  -  protected void setup (int pn, string i) -  { -  port = pn; -  ip = canonical_ip(i); -  -  restore(); -  if (sizeof(requesthandlerfile)) { -  if( file_stat( "../local/"+requesthandlerfile ) ) -  rrhf = "../local/"+requesthandlerfile; -  else -  rrhf = requesthandlerfile; -  DDUMP( rrhf ); - #ifdef DEBUG -  if( !requesthandler ) -  requesthandler = (program)(rrhf); - #endif -  } -  bound = 0; -  port_obj = 0; -  retries = 0; -  } -  -  protected void create( int pn, string i, void|int ignore_eaddrinuse ) -  //! Constructor. Bind to the port 'pn' ip 'i' -  { -  setup (pn, i); -  bind (ignore_eaddrinuse); -  } -  -  protected string _sprintf( ) -  { -  return "Protocol(" + get_url() + ")"; -  } - } -  - class InternalProtocol - //! Protocol for internal requests that are not linked to any real request. - { -  inherit Protocol; -  -  constant name = "internal"; -  -  constant prot_name = "internal"; -  -  constant supports_ipless = 1; -  constant default_port = 0; -  -  protected void create() -  { -  path = ""; -  port = default_port; -  ip = "0.0.0.0"; -  } - } -  - #if constant(SSL.File) -  - // Some convenience functions. - #if constant(SSL.Constants.fmt_cipher_suites) - constant fmt_cipher_suite = SSL.Constants.fmt_cipher_suite; - constant fmt_cipher_suites = SSL.Constants.fmt_cipher_suites; - #else - protected mapping(int:string) suite_to_symbol = ([]); -  - string fmt_cipher_suite(int suite) - { -  if (!sizeof(suite_to_symbol)) { -  foreach(indices(SSL.Constants), string id) { -  if (has_prefix(id, "SSL_") || has_prefix(id, "TLS_") || -  has_prefix(id, "SSL2_")) { -  suite_to_symbol[SSL.Constants[id]] = id; -  } -  } -  } -  string res = suite_to_symbol[suite]; -  if (res) return res; -  return suite_to_symbol[suite] = sprintf("unknown(%d)", suite); - } -  - string fmt_cipher_suites(array(int) s) - { -  String.Buffer b = String.Buffer(); -  foreach(s, int c) { -  b->add(sprintf(" %-6d: %s\n", c, fmt_cipher_suite(c))); -  } -  return (string)b; - } - #endif -  - class SSLContext { - #if constant(SSL.Context) -  inherit SSL.Context; -  - #if defined(DEBUG) || defined(SSL3_DEBUG) -  SSL.Alert alert_factory(SSL.Connection con, int level, int description, -  SSL.Constants.ProtocolVersion version, -  string|void debug_message) -  { -  if (description != SSL.Constants.ALERT_close_notify) { -  if (debug_message) { -  werror("SSL %s: %s: %s", -  (level == SSL.Constants.ALERT_warning)? -  "WARNING":"ERROR", -  SSL.Constants.fmt_constant(description, "ALERT"), -  debug_message); -  } else { -  werror("SSL %s: %s\n", -  (level == SSL.Constants.ALERT_warning)? -  "WARNING":"ERROR", -  SSL.Constants.fmt_constant(description, "ALERT")); -  } -  } -  return ::alert_factory(con, level, description, version, debug_message); -  } - #endif /* DEBUG || SSL3_DEBUG */ -  - #else -  inherit SSL.context; - #endif - } -  - //! Base protocol for protocols that support upgrading to TLS. - //! - //! Exactly like Port, but contains settings for TLS. - class StartTLSProtocol - { -  inherit Protocol; -  -  // SSL context -  SSLContext ctx = SSLContext(); -  -  int cert_failure; -  -  protected void cert_err_unbind() -  { -  if (bound > 0) { -  port_obj->close(); -  report_warning ("TLS port %s closed.\n", get_url()); -  bound = 0; -  } -  } -  - #define CERT_WARNING(VAR, MSG, ARGS...) do { \ -  string msg = (MSG); \ -  array args = ({ARGS}); \ -  if (sizeof (args)) msg = sprintf (msg, @args); \ -  report_warning ("TLS port %s: %s", get_url(), msg); \ -  (VAR)->add_warning (msg); \ -  } while (0) -  - #define CERT_ERROR(VAR, MSG, ARGS...) do { \ -  string msg = (MSG); \ -  array args = ({ARGS}); \ -  if (sizeof (args)) msg = sprintf (msg, @args); \ -  report_error ("TLS port %s: %s", get_url(), msg); \ -  (VAR)->add_warning (msg); \ -  cert_err_unbind(); \ -  cert_failure = 1; \ -  return; \ -  } while (0) -  - #if constant(SSL.Constants.PROTOCOL_TLS_MAX) -  protected void set_version() -  { -  ctx->min_version = query("ssl_min_version"); -  } - #endif -  -  protected void filter_preferred_suites() -  { - #if constant(SSL.ServerConnection) -  int mode = query("ssl_suite_filter"); -  int bits = query("ssl_key_bits"); -  -  /* Suite filter encoding: -  * -  * Bit Mask Meaning -  * 0 1 Strict suite B -  * 1 2 Transitional suite B -  * 2 4 Ephemeral only -  * 3 8 Suite B -  * 4 16 New (explicit RSA) config. -  * -  * Config value Meaning -  * 0 Default -  * 4 OLD Ephemeral key-exchanges only. -  * 8 OLD Suite B (relaxed) -  * 12 OLD Suite B (ephemeral only) -  * 14 OLD Suite B (transitional) -  * 15 OLD Suite B (strict) -  * -  * 16 Allow RSA-encryption -  * 20 Ephemeral key-exchanges only. (default) -  * 24 Suite B (allow RSA-encryption) -  * 28 Suite B (ephemeral only) -  * 30 Suite B (transitional) -  * 31 Suite B (strict) -  */ -  -  array(int) suites = ({}); -  -  if (!mode) mode = 20; // Set the default. -  -  if ((mode & 8) && !ctx->configure_suite_b) { -  // FIXME: Warn: Suite B suites not available. -  mode &= ~8; -  } -  -  if ((mode & 8) && ctx->configure_suite_b) { -  // Suite B. -  switch(mode) { -  case 15: -  // Strict mode. -  ctx->configure_suite_b(bits, 2); -  break; -  case 14: -  // Transitional mode. -  ctx->configure_suite_b(bits, 1); -  break; -  default: -  ctx->configure_suite_b(bits); -  break; -  } -  suites = ctx->preferred_suites; -  -  if (ctx->min_version < query("ssl_min_version")) { -  set_version(); -  } -  } else { -  suites = ctx->get_suites(bits, 1); -  -  // Make sure the min version is restored in case we've -  // switched from Suite B. -  set_version(); -  } -  if (mode & 4) { -  // Ephemeral suites only. -  suites = filter(suites, -  lambda(int suite) { -  return (< -  SSL.Constants.KE_dhe_dss, -  SSL.Constants.KE_dhe_rsa, -  SSL.Constants.KE_ecdhe_ecdsa, -  SSL.Constants.KE_ecdhe_rsa, -  >)[(SSL.Constants.CIPHER_SUITES[suite]||({ -1 }))[0]]; -  }); -  } -  ctx->preferred_suites = suites; - #elif constant(SSL.Constants.CIPHER_aead) -  int bits = query("ssl_key_bits"); -  // NB: The arguments to get_suites() in Pike 7.8 currently differs -  // from the ones in Pike 8.0. -  ctx->preferred_suites = ctx->get_suites(SSL.Constants.SIGNATURE_rsa, bits); - #else - #ifndef ALLOW_WEAK_SSL -  // Filter weak and really weak cipher suites. -  ctx->preferred_suites -= ({ -  SSL.Constants.SSL_rsa_with_des_cbc_sha, -  SSL.Constants.SSL_dhe_dss_with_des_cbc_sha, -  SSL.Constants.SSL_rsa_export_with_rc4_40_md5, -  SSL.Constants.TLS_rsa_with_null_sha256, -  SSL.Constants.SSL_rsa_with_null_sha, -  SSL.Constants.SSL_rsa_with_null_md5, -  SSL.Constants.SSL_dhe_dss_export_with_des40_cbc_sha, -  SSL.Constants.SSL_null_with_null_null, -  }); - #endif - #endif /* SSL.ServerConnection */ - #ifdef ROXEN_SSL_DEBUG -  report_debug("SSL: Cipher suites enabled for %O:\n" -  "%s\n", -  this_object(), -  fmt_cipher_suites(ctx->preferred_suites)); - #endif -  } -  -  void certificates_changed(Variable.Variable|void ignored, -  void|int ignore_eaddrinuse) -  { -  int old_cert_failure = cert_failure; -  cert_failure = 0; -  -  Variable.Variable Keys = getvar("ssl_keys"); -  -  array(int) keypairs = Keys->query(); -  if (!sizeof(keypairs)) { -  // No new-style certificates configured. -  -  // Check if there are old-style certificates; in case of which -  // this is probably an upgrade. -  Variable.Variable Certificates = getvar("ssl_cert_file"); -  Variable.Variable KeyFile = getvar("ssl_key_file"); -  -  keypairs = -  CertDB.register_pem_files(Certificates->query() + ({ KeyFile->query() }), -  query("ssl_password")); -  -  if (!sizeof(keypairs)) { -  // No Old-style certificate configuration found. -  // Fall back to using all known certs. -  keypairs = Keys->get_choice_list(); -  } -  -  if (sizeof(keypairs)) { -  // Certificates found. -  Keys->set(keypairs); -  -  save(); -  } else { -  // No certs known to the server. -  // Not reached except in very special circumstances. -  // FIXME: Use anonymous suites? -  report_error ("TLS port %s: %s", get_url(), -  LOC_M(63,"No certificates found.\n")); -  cert_err_unbind(); -  cert_failure = 1; -  return; -  } -  } -  -  // FIXME: Only do this if there are certs loaded? -  // We must reset the set of certificates. -  // NB: Race condition here where the new SSLContext is -  // live before it has been configured completely. -  ctx = SSLContext(); -  set_version(); -  filter_preferred_suites(); -  -  foreach(keypairs, int keypair_id) { -  array(Crypto.Sign.State|array(string)) keypair = -  CertDB.get_keypair(keypair_id); -  if (!keypair) continue; -  -  [Crypto.Sign.State private_key, array(string) certs] = keypair; -  ctx->add_cert(private_key, certs, ({ name, "*" })); -  } -  - #if 0 -  // FIXME: How do this in current Pike 8.0? -  if (!sizeof(ctx->cert_pairs)) { -  CERT_ERROR(Certificates, -  LOC_M(71,"No matching keys and certificates found.\n")); -  report_error ("TLS port %s: %s", get_url(), -  LOC_M(71,"No matching keys and certificates found.\n")); -  cert_err_unbind(); -  cert_failure = 1; -  return; -  } - #endif -  -  if (!bound) { -  bind (ignore_eaddrinuse); -  if (old_cert_failure && bound) -  report_notice (LOC_M(64, "TLS port %s opened.\n"), get_url()); -  if (!bound) -  report_notice("Failed to bind port %s.\n", get_url()); -  } -  } -  -  class CertificateKeyChoiceVariable -  { -  inherit Variable.IntChoice; -  -  mapping(int:string) get_translation_table() -  { -  array(mapping(string:int|string)) keypairs = CertDB.list_keypairs(); -  return mkmapping(keypairs->id, keypairs->name); -  } -  -  array(int) get_choice_list() -  { -  return CertDB.list_keypairs()->id; -  } -  -  array(string|mixed) verify_set(array(int) new_value) -  { -  if (!sizeof(new_value)) { -  // The list of certificates should never be empty. -  return ({ "Selection reset to all selected.", get_choice_list() }); -  } -  return ::verify_set(new_value); -  } -  -  protected mapping(Standards.ASN1.Types.Identifier:string) -  parse_dn(Standards.ASN1.Types.Sequence dn) -  { -  mapping(Standards.ASN1.Types.Identifier:string) ids = ([]); -  foreach(dn->elements, Standards.ASN1.Types.Compound pair) -  { -  if(pair->type_name!="SET" || !sizeof(pair)) continue; -  pair = pair[0]; -  if(pair->type_name!="SEQUENCE" || sizeof(pair)!=2) -  continue; -  if(pair[0]->type_name=="OBJECT IDENTIFIER" && -  pair[1]->value && !ids[pair[0]]) -  ids[pair[0]] = pair[1]->value; -  } -  return ids; -  } -  -  protected array(string) render_element(int keypair_id) -  { -  array(Crypto.Sign.State|array(string)) keypair = -  CertDB.get_keypair(keypair_id); -  if (!keypair) { -  return ({ "<td colspan='2'>" + -  LOC_C(1129, "Lost certificate") + -  "</td>" }); -  } -  [Crypto.Sign.State private_key, array(string) certs] = keypair; -  -  Standards.X509.TBSCertificate tbs = -  Standards.X509.decode_certificate(certs[0]); -  -  array(string) res = ({}); -  -  if (!tbs) { -  res += ({ "<td colspan='2'><b>" + -  LOC_C(1130, "Invalid certificate") + -  ".</b>" }); -  } else { -  mapping(Standards.ASN1.Types.Identifier:string) dn = -  parse_dn(tbs->subject); -  -  string tmp; -  if ((tmp = dn[Standards.PKCS.Identifiers.at_ids.commonName])) { -  res += ({ -  sprintf("<td style='white-space:nowrap'>%s</td>" -  "<td><b><tt>%s</tt></b>", -  LOC_C(1131, "Common Name"), -  Roxen.html_encode_string(tmp)), -  }); -  } else { -  res += ({ "<td colspan='2'>" }); -  } -  -  res[-1] += sprintf(" (%s, " + LOC_C(1132, "%d bits") + ")</td>", -  Roxen.html_encode_string(private_key->name()), -  private_key->key_size()); -  -  if (tmp = dn[Standards.PKCS.Identifiers.at_ids.organizationName]) { -  if (dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]) { -  tmp += "/" + -  dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]; -  } -  res += ({ -  sprintf("<td style='white-space:nowrap'>%s</td><td>%s</td>", -  LOC_C(1133, "Issued To"), -  Roxen.html_encode_string(tmp)), -  }); -  } else if (tmp = dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]) { -  res += ({ -  sprintf("<td style='white-space:nowrap'>%s</td><td>%s</td>", -  LOC_C(1133, "Issued To"), -  Roxen.html_encode_string(tmp)), -  }); -  } -  -  if (tbs->issuer->get_der() == tbs->subject->get_der()) { -  res += ({ -  sprintf("<td style='white-space:nowrap'>" + -  LOC_C(1134, "Issued By") + -  "</td><td>%s</td>", -  LOC_C(1135, "Self-signed")) -  }); -  } else { -  dn = parse_dn(tbs->issuer); -  tmp = dn[Standards.PKCS.Identifiers.at_ids.organizationName]; -  if (dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]) { -  tmp = (tmp?(tmp + "/"):"") + -  dn[Standards.PKCS.Identifiers.at_ids.organizationUnitName]; -  } -  string tmp2 = dn[Standards.PKCS.Identifiers.at_ids.commonName]; -  if (tmp2) { -  if (tmp) { -  tmp = tmp2 + " (" + tmp + ")"; -  } else { -  tmp = tmp2; -  } -  } -  if (tmp) { -  res += ({ -  sprintf("<td style='white-space:nowrap;vertical-align:top'>" + -  LOC_C(1134, "Issued By") + -  "</td><td>%s</td>", -  Roxen.html_encode_string(tmp)), -  }); -  } -  } -  -  tmp = Roxen.html_encode_string(Calendar.Second(tbs->not_after)-> -  format_time()); -  if (tbs->not_after < time(1)) { -  // Already expired. -  res += ({ -  sprintf("<td>%s</td>" -  "<td><font color='&usr.warncolor;'>%s</font>\n" -  "<img src='&usr.err-3;' /></td>", -  LOC_C(1136, "Expired"), -  tmp), -  }); -  } else if (tbs->not_after < time(1) + (3600 * 24 * 30)) { -  // Expires within 30 days. -  res += ({ -  sprintf("<td>%s</td>" -  "<td><font color='&usr.warncolor;'>%s</font>\n" -  "<img src='&usr.err-2;' /></td>", -  LOC_C(1137, "Expires"), -  tmp), -  }); -  } else { -  res += ({ -  sprintf("<td>%s</td><td>%s</td>", LOC_C(1137, "Expires"), tmp), -  }); -  } -  -  mapping keypair_metadata = CertDB.get_keypair_metadata(keypair_id); -  -  array(string) paths = -  keypair_metadata->certs->pem_path + -  ({ keypair_metadata->key->pem_path }); -  paths = Array.uniq(paths); -  paths = replace(paths, 0, "__LOST__"); -  object privs = Privs("Scanning directory for pem files."); -  paths = map(paths, lfile_path); -  privs = 0; -  res += ({ -  sprintf("<td style='vertical-align:top'>%s</td><td>%s</td>", -  LOC_C(1138, "Path(s)"), -  map(paths, lambda(string p) { -  if (p) -  return "<tt>" + Roxen.html_encode_string(p) + "</tt>"; -  else -  return -  "<font color='&usr.warncolor;'>" + -  LOC_C(1139, "Lost file") + -  "</font>"; -  }) * "<br/>") -  }); -  } -  -  return res; -  } -  -  string render_form(RequestID id, void|mapping additional_args) { -  array(string) current = map(query(), _name); -  string res = "<table width='100%'>\n"; -  foreach( get_choice_list(); int i; mixed elem ) { -  if (i != 0) { -  res += "<tr><td colspan='3'><hr/></td></tr>\n"; -  } -  mapping m = ([ -  "type": "checkbox", -  "name": path(), -  "value": _name(elem), -  ]); -  if(has_value(current, m->value)) { -  m->checked="checked"; -  current -= ({ m->value }); -  } -  array(string) el_rows = render_element(elem); -  res += sprintf("<tr><td rowspan='%d'>%s</td>" -  "%s" -  "</tr>\n", -  sizeof(el_rows), -  Roxen.make_tag( "input", m), -  el_rows[0]); -  foreach(el_rows[1..], string row) { -  res += sprintf("<tr>%s</tr>", row); -  } -  } -  // Make an entry for the current values if they're not in the list, -  // to ensure that the value doesn't change as a side-effect by -  // another change. -  foreach(current, string value) { -  mapping m = ([ -  "type": "checkbox", -  "name": path(), -  "value": value, -  "checked": "checked", -  ]); -  string title = sprintf(LOC_C(1121,"(stale value %s)"), value); -  res += sprintf("<tr><td>%s</td><td>%s</td></tr>\n", -  Roxen.make_tag( "input", m), -  Roxen.html_encode_string(title)); -  } -  return res + "</table>"; -  } -  -  protected void create( void|int _flags, void|LocaleString std_name, -  void|LocaleString std_doc ) -  { -  ::create(({}), UNDEFINED, _flags, std_name, std_doc); -  } -  } -  - #if 1 -  // Old-style SSL Certificate variables. -  // FIXME: Keep these around for at least a few major versions (10 years?). -  class CertificateListVariable -  { -  inherit Variable.FileList; -  -  string doc() -  { -  return sprintf(::doc() + "\n", -  combine_path(getcwd(), "../local"), -  getcwd()); -  } -  } -  -  class KeyFileVariable -  { -  inherit Variable.String; -  -  string doc() -  { -  return sprintf(::doc() + "\n", -  combine_path(getcwd(), "../local"), -  getcwd()); -  } -  } - #endif -  -  void create(int pn, string i, void|int ignore_eaddrinuse) -  { -  ctx->random = Crypto.Random.random_string; -  -  set_up_ssl_variables( this_object() ); -  -  // NB: setup() calls restore() which initializes the variables -  // created above. -  ::setup(pn, i); -  - #if constant(SSL.Constants.PROTOCOL_TLS_MAX) -  set_version(); - #endif -  -  filter_preferred_suites(); -  -  certificates_changed (0, ignore_eaddrinuse); -  -  // Install the change callbacks here to avoid duplicate calls -  // above. -  // FIXME: Both variables ought to be updated on save before the -  // changed callback is called. Currently you can get warnings -  // that the files don't match if you update both variables -  // at the same time. -  getvar ("ssl_keys")->set_changed_callback(certificates_changed); -  getvar ("ssl_cert_file")->set_changed_callback (certificates_changed); -  getvar ("ssl_key_file")->set_changed_callback (certificates_changed); -  - #if constant(SSL.Constants.CIPHER_aead) -  getvar("ssl_key_bits")->set_changed_callback(filter_preferred_suites); - #endif - #if constant(SSL.ServerConnection) -  getvar("ssl_suite_filter")->set_changed_callback(filter_preferred_suites); - #endif - #if constant(SSL.Constants.PROTOCOL_TLS_MAX) -  getvar("ssl_min_version")->set_changed_callback(set_version); - #endif -  } -  -  string _sprintf( ) -  { -  return "StartTLSProtocol(" + get_url() + ")"; -  } - } -  - class SSLProtocol - //! Base protocol for SSL ports. - //! - //! Exactly like Port, but uses SSL. - { -  inherit StartTLSProtocol; -  -  SSL.File accept() -  { -  Stdio.File q = ::accept(); -  if (q) { -  SSL.File ssl = SSL.File(q, ctx); -  ssl->accept(); -  return ssl; -  } -  return 0; -  } -  - #if constant(SSL.Connection) -  protected void bind (void|int ignore_eaddrinuse) -  { -  // Don't bind if we don't have correct certs. -  // if (!sizeof(ctx->cert_pairs)) return; -  ::bind (ignore_eaddrinuse); -  } - #else -  protected void bind (void|int ignore_eaddrinuse) -  { -  // Don't bind if we don't have correct certs. -  if (!ctx->certificates) return; -  ::bind (ignore_eaddrinuse); -  } - #endif -  -  string _sprintf( ) -  { -  return "SSLProtocol(" + get_url() + ")"; -  } - } - #endif -  - mapping(string:program/*(Protocol)*/) build_protocols_mapping() - { -  mapping protocols = ([]); -  int st = gethrtime(); -  report_debug("Protocol handlers ... \b"); - #ifndef DEBUG -  class lazy_load( string prog, string name ) -  { -  program real; -  protected void realize() -  { -  if( catch { -  DDUMP( prog ); -  real = (program)prog; -  protocols[name] = real; -  } ) -  report_error("Failed to compile protocol handler for "+name+"\n"); -  } -  -  Protocol `()(mixed ... x) -  { -  if(!real) realize(); -  return real(@x); -  }; -  mixed `->( string x ) -  { -  if(!real) realize(); -  return predef::`->(real, x); -  } -  }; - #endif -  foreach( glob( "prot_*.pike", get_dir("protocols") ), string s ) -  { -  sscanf( s, "prot_%s.pike", s ); - #if !constant(SSL.File) -  switch( s ) -  { -  case "https": -  case "ftps": -  continue; -  } - #endif -  report_debug( "\b%s \b", s ); -  -  catch -  { - #ifdef DEBUG -  protocols[ s ] = (program)("protocols/prot_"+s+".pike"); - #else -  protocols[ s ] = lazy_load( ("protocols/prot_"+s+".pike"),s ); - #endif -  }; -  } -  foreach( glob("prot_*.pike",get_dir("../local/protocols")||({})), string s ) -  { -  sscanf( s, "prot_%s.pike", s ); - #if !constant(SSL.File) -  switch( s ) -  { -  case "https": -  case "ftps": -  continue; -  } - #endif -  report_debug( "\b%s \b", s ); -  catch { - #ifdef DEBUG -  protocols[ s ] = (program)("../local/protocols/prot_"+s+".pike"); - #else -  protocols[ s ] = lazy_load( ("../local/protocols/prot_"+s+".pike"),s ); - #endif -  }; -  } -  report_debug("\bDone [%.1fms]\n", (gethrtime()-st)/1000.0 ); -  return protocols; - } -  -  - mapping(string:program/*(Protocol)*/) protocols; -  - //! Lookup from protocol, IP number and port to - //! the corresponding open @[Protocol] port. - //! - //! @mapping - //! @member mapping(string:mapping(int:Protocol)) protocol_name - //! @mapping - //! @member mapping(int:Protocol) ip_number - //! @mapping - //! @member Protocol port_number - //! @[Protocol] object that holds this ip_number and port open. - //! @endmapping - //! @endmapping - //! @endmapping - mapping(string:mapping(string:mapping(int:Protocol))) open_ports = ([ ]); -  - //! Lookup from URL string to the corresponding open @[Protocol] ports. - //! - //! Note that there are two classes of URL strings used as indices in - //! this mapping: - //! @dl - //! @item "prot://host_glob:port/path/" - //! A normalized URL string as returned by @[normalize_url()]. - //! - //! @[Protocol()->ref()] in the contained ports as been called - //! with the url. - //! - //! @item "port://host_glob:port/path/#opt1=val1;opt2=val2" - //! An URL string containing options as stored in the @tt{"URLs"@} - //! configuration variable, and expected as argument by - //! @[register_url()] and @[unregister_url()]. Also known - //! as an ourl. - //! @enddl - //! - //! In both cases the same set of data is stored: - //! @mapping - //! @member mapping(string:Configuration|Protocol|string|array(Protocol)|array(string)) url - //! @mapping - //! @member Protocol "port" - //! Representative open port for this URL. - //! @member array(Protocol) "ports" - //! Array of all open ports for this URL. - //! @member Configuration "conf" - //! Configuration that has registered the URL. - //! @member string "path" - //! Path segment of the URL. - //! @member string "host" - //! Hostname segment of the URL. - //! @member array(string) "skipped" - //! List of IP numbers not bound due to a corresponding - //! ANY port already being open. - //! @endmapping - //! @endmapping - mapping(string:mapping(string:Configuration|Protocol|string|array(Protocol)|array(string))) -  urls = ([]); -  - array sorted_urls = ({}); -  - array(string) find_ips_for( string what ) - { -  if( what == "*" || lower_case(what) == "any" || has_value(what, "*") ) -  return ({ - #if constant(__ROXEN_SUPPORTS_IPV6__) -  "::", - #endif /* __ROXEN_SUPPORTS_IPV6__ */ -  0, -  }); // ANY -  -  if( is_ip( what ) ) -  return ({ what }); -  else if (has_prefix (what, "[") && what[-1] == ']') { -  /* RFC 3986 3.2.2. Host -  * -  * host = IP-literal / IPv4address / reg-name -  * IP-literal = "[" ( IPv6address / IPvFuture ) "]" -  * IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ) -  * -  * IPv6address is as in RFC3513. -  */ -  return ({ what[1..sizeof(what)-2] }); -  } else if (has_suffix(lower_case(what), ".ipv6")) { -  // draft-masinter-url-ipv6-00 3 -  // -  // a) replace every colon ":" with a "-" -  // b) append ".ipv6" to the end. -  return ({ replace(what[..sizeof(what)-6], "-", ":") }); -  } -  array res = gethostbyname( what ); -  if( res && sizeof( res[1] ) ) -  return Array.uniq(res[1]); -  -  report_error(LOC_M(46, "Cannot possibly bind to %O, that host is " -  "unknown. Substituting with ANY")+"\n", what); -  return 0; // FAIL - } -  - string normalize_url(string url, void|int port_match_form) - //! Normalizes the given url to a short form. - //! - //! If @[port_match_form] is set, it normalizes to the form that is - //! used for port matching, i.e. what - //! @[roxen.Protocol.find_configuration_for_url] expects. - //! - //! @note - //! Returns @expr{""@} for @[url]s that are incomplete. - { -  if (!sizeof (url - " " - "\t")) return ""; -  -  Standards.URI ui = Standards.URI(url); -  string host = ui->host; -  -  if (lower_case (host) == "any" || host == "::") -  host = "*"; -  else { -  // Note: zone_to_ascii() can throw errors on invalid hostnames. -  if (catch { -  // FIXME: Maybe Standards.URI should do this internally? -  host = lower_case(Standards.IDNA.zone_to_ascii (host)); -  }) { -  host = lower_case(host); -  } -  } -  -  if (has_value (host, ":")) -  if (string h = port_match_form ? -  Protocols.IPv6.normalize_addr_basic (host) : -  Protocols.IPv6.normalize_addr_short (host)) { -  ui->host = h; -  host = "[" + h + "]"; -  } -  -  string protocol = ui->scheme; -  if (host == "" || !protocols[protocol]) -  return ""; -  -  if (port_match_form) { -  int port = ui->port || protocols[protocol]->default_port; -  -  string path = ui->path; -  if (path) { -  if (has_suffix(path, "/")) -  path = path[..<1]; -  } -  else -  path = ""; -  -  return sprintf ("%s://%s:%d%s/", protocol, host, port, -  // If the path is set it's assumed to begin with a -  // "/", but not end with one. -  path); -  } -  -  else { -  ui->fragment = 0; -  return (string) ui; -  } - } -  - //! Unregister an URL from a configuration. - //! - //! @seealso - //! @[register_url()] - void unregister_url(string url, Configuration conf) - { -  string ourl = url; -  mapping(string:mixed) data = m_delete(urls, ourl); -  if (!data) return; // URL not registered. -  if (!sizeof(url = normalize_url(url, 1))) return; -  -  report_debug ("Unregister %s%s.\n", normalize_url (ourl), -  conf ? sprintf (" for %O", conf->query_name()) : ""); -  -  mapping(string:mixed) shared_data = urls[url]; -  if (!shared_data) return; // Strange case, but URL not registered. -  -  int was_any_ip; -  if (!data->skipped && data->port) { -  if (!data->port->ip || (data->port->ip == "::")) { -  was_any_ip = data->port->port; -  report_debug("Unregistering ANY port: %O:%d\n", -  data->port->ip, data->port->port); -  } -  } -  -  foreach(data->ports || ({}), Protocol port) { -  shared_data->ports -= ({ port }); -  port->unref(url); -  m_delete(shared_data, "port"); -  } -  if (!sizeof(shared_data->ports || ({}))) { -  m_delete(urls, url); -  } else if (!shared_data->port) { -  shared_data->port = shared_data->ports[0]; -  } -  sort_urls(); -  -  if (was_any_ip) { -  foreach(urls; string url; mapping(string:mixed) url_info) { -  if (!url_info->skipped || !url_info->conf || -  (url_info->port && (url_info->port->port != was_any_ip))) { -  continue; -  } -  // Re-register the ports that may have bound to the removed ANY port. -  register_url(url, url_info->conf); -  } -  } - } -  - array all_ports( ) - { -  // FIXME: Consider using open_ports instead. -  return Array.uniq( (values( urls )->ports - ({0})) * ({}) )-({0}); - } -  - Protocol find_port( string name ) - { -  foreach( all_ports(), Protocol p ) -  if( p->get_key() == name ) -  return p; - } -  - void sort_urls() - { -  sorted_urls = indices( urls ); -  sort( map( map( sorted_urls, strlen ), `-), sorted_urls ); - } -  - //! Register an URL for a configuration. - //! - //! @seealso - //! @[unregister_url()] - int register_url( string url, Configuration conf ) - { -  string ourl = url; -  if (!sizeof (url - " " - "\t")) return 1; -  -  Standards.URI ui = Standards.URI(url); -  mapping opts = ([]); -  string a, b; -  foreach( (ui->fragment||"")/";", string x ) -  { -  sscanf( x, "%s=%s", a, b ); -  opts[a]=b; -  } -  -  if( (int)opts->nobind ) -  { -  report_warning( -  LOC_M(61,"Not binding the port %O - disabled in configuration.")+"\n", -  (string) ui ); -  return 0; -  } -  -  string display_url = normalize_url (url, 0); -  url = normalize_url (url, 1); -  if (url == "") return 1; -  ui = Standards.URI (url); -  -  string protocol = ui->scheme; -  string host = ui->host; -  if (host == "" || !protocols[protocol]) { -  report_error(LOC_M(19,"Bad URL %O for server %O.")+"\n", -  ourl, conf->query_name()); -  } -  -  int port = ui->port || protocols[protocol]->default_port; -  -  string path = ui->path; -  if (has_suffix(path, "/")) -  path = path[..<1]; -  if (path == "") path = 0; -  -  if( urls[ url ] ) -  { -  if( !urls[ url ]->port ) -  m_delete( urls, url ); -  else if( urls[ url ]->conf ) -  { -  if( urls[ url ]->conf != conf ) -  { -  report_error(LOC_M(20, -  "Cannot register URL %s - " -  "already registered by %s.")+"\n", -  display_url, urls[ url ]->conf->name); -  return 0; -  } -  // FIXME: Is this correct? -  urls[ url ]->port->ref(url, urls[url]); -  } -  else -  urls[ url ]->port->unref( url ); -  } -  -  program prot; -  -  if( !( prot = protocols[ protocol ] ) ) -  { -  report_error(LOC_M(21, "Cannot register URL %s - " -  "cannot find the protocol %s.")+"\n", -  display_url, protocol); -  return 0; -  } -  -  // FIXME: Do we need to unref the old ports first in case of a reregister? -  urls[ ourl ] = ([ "conf":conf, "path":path, "hostname": host ]); -  if (!urls[url]) { -  urls[ url ] = urls[ourl] + ([]); -  sorted_urls += ({ url }); // FIXME: Not exactly sorted... -  } -  -  array(string)|int(-1..0) required_hosts; -  -  if (is_ip(host)) -  required_hosts = ({ host }); -  else if(!sizeof(required_hosts = -  filter(replace(opts->ip||"", " ","")/",", is_ip)) ) { -  required_hosts = find_ips_for( host ); -  if (!required_hosts) { -  // FIXME: Used to fallback to ANY. -  // Will this work with glob URLs? -  return 0; -  } -  } -  -  mapping(string:mapping(int:Protocol)) m; -  if( !( m = open_ports[ protocol ] ) ) -  // always add 'ANY' (0) and 'IPv6_ANY' (::) here, as empty mappings, -  // for speed reasons. -  // There is now no need to check for both open_ports[prot][0] and -  // open_ports[prot][0][port], we can go directly to the latter -  // test. -  m = open_ports[ protocol ] = ([ 0:([]), "::":([]) ]); -  -  if (prot->supports_ipless ) { -  // Check if the ANY port is already open for this port, since this -  // protocol supports IP-less virtual hosting, there is no need to -  // open yet another port if it is, since that would most probably -  // only conflict with the ANY port anyway. (this is true on most -  // OSes, it works on Solaris, but fails on linux) -  array(string) ipv6 = filter(required_hosts - ({ 0 }), has_value, ":"); -  array(string) ipv4 = required_hosts - ipv6; -  if (m[0][port] && sizeof(ipv4 - ({ 0 }))) { -  // We have a non-ANY IPv4 IP number. -  // Keep track of the ips in case the ANY port is removed. -  urls[ourl]->skipped = ipv4; -  ipv4 = ({ 0 }); -  } - #if constant(__ROXEN_SUPPORTS_IPV6__) -  if (m["::"][port] && sizeof(ipv6 - ({ "::" }))) { -  // We have a non-ANY IPv6 IP number. -  // Keep track of the ips in case the ANY port is removed. -  urls[ourl]->skipped += ipv6; -  ipv6 = ({ "::" }); -  } -  required_hosts = ipv6 + ipv4; - #else -  if (sizeof(ipv6)) { -  foreach(ipv6, string p) { -  report_warning(LOC_M(65, "Cannot open port %s for URL %s - " -  "IPv6 support disabled.\n"), -  p, display_url); -  } -  } -  required_hosts = ipv4; - #endif /* __ROXEN_SUPPORTS_IPV6__ */ -  } -  -  int failures; -  int opened_ipv6_any_port; -  -  foreach(required_hosts, string required_host) -  { -  if( m[ required_host ] && m[ required_host ][ port ] ) -  { -  if (required_host == "::") opened_ipv6_any_port = 1; -  -  m[required_host][port]->ref(url, urls[url]); -  -  urls[url]->port = m[required_host][port]; -  if (urls[url]->ports) { -  urls[url]->ports += ({ m[required_host][port] }); -  } else { -  urls[url]->ports = ({ m[required_host][port] }); -  } -  if (ourl != url) { -  urls[ourl]->port = m[required_host][port]; -  if (urls[ourl]->ports) { -  urls[ourl]->ports += ({ m[required_host][port] }); -  } else { -  urls[ourl]->ports = ({ m[required_host][port] }); -  } -  } -  continue; /* No need to open a new port */ -  } -  -  if( !m[ required_host ] ) -  m[ required_host ] = ([ ]); -  -  -  Protocol prot_obj; -  if (mixed err = catch { -  prot_obj = m[ required_host ][ port ] = -  prot( port, required_host, -  // Don't complain if binding IPv4 ANY fails with -  // EADDRINUSE after we've bound IPv6 ANY. -  // Most systems seems to bind both IPv4 ANY and -  // IPv6 ANY for "::" -  !required_host && opened_ipv6_any_port); -  }) { -  failures++; - #if 0 -  if (has_prefix(describe_error(err), "Invalid address") && -  required_host && has_value(required_host, ":")) { -  report_error(sprintf("Failed to initialize IPv6 port for URL %s" -  " (ip %s).\n", -  display_url, required_host)); -  } else - #endif -  report_error(sprintf("Initializing the port handler for " -  "URL %s (ip %s) failed: %s\n", -  display_url, -  required_host||"ANY", - #ifdef DEBUG -  describe_backtrace(err) - #else -  describe_error (err) - #endif -  )); -  continue; -  } -  -  if (required_host == "::") opened_ipv6_any_port = 1; -  -  if (prot_obj->bound == -1) { -  // Got EADDRINUSE for the IPv6 case - see above. Just forget -  // about this one. -  m_delete (m[required_host], port); -  continue; -  } -  -  urls[ url ]->port = prot_obj; -  if (urls[url]->ports) { -  urls[url]->ports += ({ prot_obj }); -  } else { -  urls[url]->ports = ({ prot_obj }); -  } -  if (ourl != url) { -  urls[ ourl ]->port = prot_obj; -  if (urls[ourl]->ports) { -  urls[ourl]->ports += ({ prot_obj }); -  } else { -  urls[ourl]->ports = ({ prot_obj }); -  } -  } -  prot_obj->ref(url, urls[url]); -  -  if( !prot_obj->bound ) -  failures++; -  } -  if (failures == sizeof(required_hosts)) -  { -  report_error(LOC_M(23, "Failed to register URL %s for %O.")+"\n", -  display_url, conf->query_name()); -  return 0; -  } -  sort_urls(); -  -  // The following will show the punycoded version for IDN hostnames. -  // That is intentional, to make it clear what actually happens. -  report_notice(" "+LOC_S(3, "Registered URL %s for %O.")+"\n", -  display_url, conf->query_name() ); -  return 1; - } -  -  - Configuration find_configuration( string name ) - //! Searches for a configuration with a name or fullname like the - //! given string. See also get_configuration(). - { -  // Optimization, in case the exact name is given... -  if( Configuration o = get_configuration( name ) ) -  return o; -  -  name = replace( lower_case( replace(name,"-"," ") )-" ", "/", "-" ); -  foreach( configurations, Configuration o ) -  { -  if( (lower_case( replace( replace(o->name, "-"," ") - " " , -  "/", "-" ) ) == name) || -  (lower_case( replace( replace(o->query_name(), "-", " ") - " " , -  "/", "-" ) ) == name) ) -  return o; -  } -  return 0; - } -  - protected int last_hrtime = gethrtime(1)/100; - protected int clock_sequence = random(0x4000); - protected string hex_mac_address = -  String.string2hex(Crypto.Random.random_string (6)| -  "\1\0\0\0\0\0"); // Multicast bit. - // Generate an uuid string. - string new_uuid_string() - { -  int now = gethrtime(1)/100; -  if (now != last_hrtime) { -  clock_sequence = random(0x4000); -  last_hrtime = now; -  } -  int seq = clock_sequence++; -  // FIXME: Check if clock_sequence has wrapped during this @[now]. -  -  // Adjust @[now] with the number of 100ns intervals between -  // 1582-10-15 00:00:00.00 GMT and 1970-01-01 00:00:00.00 GMT. - #if 0 -  now -= Calendar.parse("%Y-%M-%D %h:%m:%s.%f %z", -  "1582-10-15 00:00:00.00 GMT")->unix_time() * 10000000; - #else /* !0 */ -  now += 0x01b21dd213814000; // Same as above. - #endif /* 0 */ -  now &= 0x0fffffffffffffff; -  now |= 0x1000000000000000; // DCE version 1. -  clock_sequence &= 0x3fff; -  clock_sequence |= 0x8000; // DCE variant of UUIDs. -  return sprintf("%08x-%04x-%04x-%04x-%s", -  now & 0xffffffff, -  (now >> 32) & 0xffff, -  (now >> 48) & 0xffff, -  clock_sequence, -  hex_mac_address); - } -  - mapping(string:array(int)) error_log=([]); -  - // Write a string to the administration interface error log and to stderr. - void nwrite(string s, int|void perr, int|void errtype, -  object|void mod, object|void conf) - { -  int log_time = time(1); -  string reference = (mod ? Roxen.get_modname(mod) : conf && conf->name) || ""; -  string log_index = sprintf("%d,%s,%s", errtype, reference, s); -  if(!error_log[log_index]) -  error_log[log_index] = ({ log_time }); -  else -  error_log[log_index] += ({ log_time }); -  -  if( mod ) -  { -  if( mod->error_log ) -  mod->error_log[log_index] += ({ log_time }); -  } -  if( conf ) -  { -  if( conf->error_log ) -  conf->error_log[log_index] += ({ log_time }); -  } -  -  if(errtype >= 1) -  report_debug( s ); - } -  - protected BackgroundProcess hourly_maintenance_process; -  - protected void clean_error_log(mapping(string:array(int)) log, -  mapping(string:int) cutoffs) - { -  if (!log || !sizeof(log)) return; -  foreach(cutoffs; string prefix; int cutoff) { -  foreach(log; string key; array(int) times) { -  if (!has_prefix(key, prefix)) continue; -  int sz = sizeof(times); -  times = filter(times, `>=, cutoff); -  if (sizeof(times) == sz) continue; -  // NB: There's a race here, where newly triggered errors may be lost. -  // It's very unlikely to be a problem in practice though. -  if (!sizeof(times)) { -  m_delete(log, key); -  } else { -  log[key] = times; -  } -  } -  } - } -  - protected void error_log_cleaner() - { -  mapping(string:int) cutoffs = ([ -  "1,": time(1) - 3600*24*7, // Keep notices for 7 days. -  ]); -  -  // First the global error_log. -  clean_error_log(error_log, cutoffs); -  -  // Then all configurations and modules. -  foreach(configurations, Configuration conf) { -  clean_error_log(conf->error_log, cutoffs); -  -  foreach(indices(conf->otomod), RoxenModule mod) { -  clean_error_log(mod->error_log, cutoffs); -  } -  } - } -  - protected void patcher_report_notice(string msg, mixed ... args) - { -  if (sizeof(args)) msg = sprintf(msg, @args); -  report_notice(RoxenPatch.wash_output(msg)); - } -  - protected void patcher_report_error(string msg, mixed ... args) - { -  if (sizeof(args)) msg = sprintf(msg, @args); -  report_error(RoxenPatch.wash_output(msg)); - } -  - RoxenPatch.Patcher plib = -  RoxenPatch.Patcher(patcher_report_notice, patcher_report_error, -  getcwd(), getenv("LOCALDIR")); -  - protected void hourly_maintenance() - { -  error_log_cleaner(); -  -  if (query("auto_fetch_rxps")) { -  plib->import_file_http(); -  } - } -  - protected void start_hourly_maintenance() - { -  if (hourly_maintenance_process) return; -  -  // Start a background process that performs maintenance tasks every hour -  // (eg cleaning the error log). -  hourly_maintenance_process = BackgroundProcess(3600, hourly_maintenance); - } -  - protected void stop_hourly_maintenance() - { -  if (hourly_maintenance_process) { -  hourly_maintenance_process->stop(); -  hourly_maintenance_process = UNDEFINED; -  } - } -  - // When was Roxen started? - int boot_time =time(); - int start_time =time(); -  - string version() - { - #ifndef NSERIOUS -  return query("default_ident")?real_version:query("ident"); - #else -  multiset choices=(<>); -  string version=query("default_ident")?real_version:query("ident"); -  return version+", "+ ({ -  "Applier of Templates", -  "Beautifier of Layouts", -  "Conqueror of Comdex", -  "Deliverer of Documents", -  "Enhancer of Abilities", -  "Freer of Webmasters", -  "Generator of Logs", -  "Helper of Users", -  "Interpreter of Scripts", -  "Juggler of Java-code", -  "Keeper of Databases", -  "Locator of Keywords", -  "Manipulator of Data", -  "Negatiator of Protocols", -  "Operator of Sites", -  "Provider of Contents", -  "Quintessence of Quality", -  "Responder to Connections", -  "Server of Webs", -  "Translator of Texts", -  "Unifier of Interfaces", -  "Valet of Visitors", -  "Watcher for Requests", -  "Xylem of Services", -  "Yielder of Information", -  "Zenith of Extensibility" -  })[random(26)]; - #endif - } -  - public void log(mapping file, RequestID request_id) - { -  if(!request_id->conf) return; -  request_id->conf->log(file, request_id); - } -  - #if ROXEN_COMPAT < 2.2 - // Support for unique user id's - private Stdio.File current_user_id_file; - private int current_user_id_number, current_user_id_file_last_mod; -  - private void restore_current_user_id_number() - { -  if(!current_user_id_file) -  current_user_id_file = open(configuration_dir + "LASTUSER~", "rwc"); -  if(!current_user_id_file) -  { -  call_out(restore_current_user_id_number, 2); -  return; -  } -  current_user_id_number = (int)current_user_id_file->read(100); -  current_user_id_file_last_mod = current_user_id_file->stat()[2]; -  report_debug("Restoring unique user ID information. (" + current_user_id_number -  + ")\n"); - #ifdef FD_DEBUG -  mark_fd(current_user_id_file->query_fd(), "Unique user ID logfile.\n"); - #endif - } -  - int increase_id() - { -  if(!current_user_id_file) -  { -  restore_current_user_id_number(); -  return current_user_id_number+time(1); -  } -  if(current_user_id_file->stat()[2] != current_user_id_file_last_mod) -  restore_current_user_id_number(); -  current_user_id_number++; -  current_user_id_file->seek(0); -  current_user_id_file->write((string)current_user_id_number); -  current_user_id_file_last_mod = current_user_id_file->stat()[2]; -  return current_user_id_number; - } - #endif // ROXEN_COMPAT < 2.2 -  - private int unique_id_counter; - string create_unique_id() - { -  return String.string2hex(Crypto.MD5.hash(query("server_salt") + start_time -  + "|" + (unique_id_counter++) + "|" + time(1))); - } -  - #ifndef __NT__ - protected int abs_started; - protected int handlers_alive; -  - protected void low_engage_abs() - { -  report_debug("**** %s: ABS exiting roxen!\n\n", -  ctime(time()) - "\n"); -  _exit(1); // It might not quit correctly otherwise, if it's -  // locked up. Note that this also inhibits the delay -  // caused by the possible automatic installation of -  // any pending patches. - } -  - protected void engage_abs(int n) - { -  if (!query("abs_engage")) { -  abs_started = 0; -  report_debug("Anti-Block System Disabled.\n"); -  return; -  } -  report_debug("**** %s: ABS engaged!\n" -  "Waited more than %d minute(s).\n", -  ctime(time()) - "\n", -  query("abs_timeout")); -  // Paranoia exit in case describe_all_threads below hangs. -  signal(signum("SIGALRM"), low_engage_abs); -  int t = alarm(20); -  report_debug("\nTrying to dump backlog: \n"); -  if (mixed err = catch { -  // Catch for paranoia reasons. -  describe_all_threads(); -  }) -  master()->handle_error (err); - #ifdef THREADS -  report_debug("\nHandler queue:\n"); -  if (mixed err = catch { -  t = alarm(20); // Restart the timeout timer. -  array(mixed) queue = handle_queue->buffer[handle_queue->r_ptr..]; -  foreach(queue, mixed v) { -  if (!v) { -  // Either an entry past the write pointer, or an entry that -  // has been zapped by a handler thread during our processing. -  continue; -  } -  if (!arrayp(v)) { -  report_debug(" *** Strange entry: %O ***\n", v); -  } else { -  report_debug(" %{%O, %}\n", v/({})); -  } -  } -  }) -  master()->handle_error (err); - #endif -  report_debug("\nPending call_outs:\n"); -  if (mixed err = catch { -  t = alarm(20); // Restart the timeout timer. -  foreach(call_out_info(), array info) { -  report_debug(" %4d seconds: %O(%{%O, %})\n", -  info[0], info[2], info[3..]); -  } -  }) -  master()->handle_error(err); -  low_engage_abs(); - } -  - //! Called to indicate that the handler threads are alive. - //! - //! Usually called automatically via @[handle()] by @[restart_if_stuck()], - //! but may need to be called by hand when handler threads are intentionally - //! blocked for a longer time (eg via long-lived global mutex locks). - //! - //! Do not call unless you know what you are doing. - void handler_ping() - { -  handlers_alive = time(); - } -  - void restart_if_stuck (int force) - //! @note - //! Must be called from the backend thread due to Linux peculiarities. - { -  remove_call_out(restart_if_stuck); -  if (!(query("abs_engage") || force)) -  return; -  if(!abs_started) -  { -  abs_started = 1; -  handlers_alive = time(); -  report_debug("Anti-Block System Enabled.\n"); -  } -  call_out (restart_if_stuck,10); - // werror("call_out_info %O\n", filter(call_out_info(), lambda(array a) { - // return a[2] == restart_if_stuck; })); -  signal(signum("SIGALRM"), engage_abs); -  int t = alarm (60*query("abs_timeout")+20); -  // werror("alarm: %d seconds left, set to %d\n", t, 60*query("abs_timeout")+20); -  if ((time(1) - handlers_alive) > 60*query("abs_timeout")) { -  // The handler installed below hasn't run. -  report_debug("**** %s: ABS: Handlers are dead!\n", -  ctime(time()) - "\n"); -  engage_abs(0); -  } -  handle(handler_ping); - } - #endif -  - #if constant(ROXEN_MYSQL_SUPPORTS_UNICODE) - // NOTE: We need to mark binary data as binary in case - // the Mysql character_set_connection has been - // set to anything other than "latin1". - #define MYSQL__BINARY "_binary" - #else - #define MYSQL__BINARY "" - #endif -  - function(string:Sql.Sql) dbm_cached_get; -  - // Threshold in seconds for updating atime records Currently set to - // one day. - #define ATIME_THRESHOLD 60*60*24 -  - class ImageCache - //! The image cache handles the behind-the-scenes caching and - //! regeneration features of graphics generated on the fly. Besides - //! being a cache, however, it serves a wide variety of other - //! interesting image conversion/manipulation functions as well. - { - #define QUERY(X,Y...) get_db()->query(X,Y) -  string name; -  string dir; -  function draw_function; -  mapping(string:array(mapping|int)) meta_cache = ([]); -  -  string documentation(void|string tag_n_args) -  { -  string doc = Stdio.read_file("base_server/image_cache.xml"); -  if(!doc) return ""; -  if(!tag_n_args) -  return Parser.HTML()->add_container("ex", "")-> -  add_quote_tag("!--","","--")->finish(doc)->read(); -  return replace(doc, "###", tag_n_args); -  } -  -  protected mapping meta_cache_insert( string i, mapping what ) -  { - #ifdef ARG_CACHE_DEBUG -  werror("MD insert for %O: %O\n", i, what ); - #endif -  if( sizeof( meta_cache ) > 1000 ) -  sync_meta(); -  if( what ) { -  meta_cache[i] = ({ what, 0 }); -  return what; -  } -  else -  m_delete( meta_cache, i ); -  return 0; -  } -  -  protected mixed frommapp( mapping what ) -  { -  if( !what ) -  error( "Got invalid argcache-entry\n" ); -  if( !zero_type(what[""]) ) return what[""]; -  return what; -  } -  -  protected void|mapping draw( string name, RequestID id ) -  { - #ifdef ARG_CACHE_DEBUG -  werror("draw: %O id: %O\n", name, id ); - #endif -  mixed args = Array.map( Array.map( name/"$", argcache->lookup, -  id->client ), frommapp); -  -  id->cache_status["icachedraw"] = 1; -  -  mapping meta; -  string data; -  array guides; - #ifdef ARG_CACHE_DEBUG -  werror("draw args: %O\n", args ); - #endif -  mixed reply = draw_function( @copy_value(args), id ); -  -  if( !reply ) { - #ifdef ARG_CACHE_DEBUG -  werror("%O(%{%O, %}%O) ==> 0\n", -  draw_function, args, id); - #endif -  return; -  } -  -  if( arrayp( args ) ) -  args = args[0]; -  -  if( arrayp( reply ) ) // layers. -  { -  guides = reply->get_misc_value( "image_guides" )-({}); -  if( sizeof( guides ) ) -  guides = guides[0]; -  reply = Image.lay( reply ); -  } -  if( objectp( reply ) && reply->image ) // layer. -  { -  if( !guides ) -  guides = reply->get_misc_value( "image_guides" ); -  reply = ([ -  "img":reply->image(), -  "alpha":reply->alpha(), -  ]); -  } -  -  -  if( objectp( reply ) || (mappingp(reply) && reply->img) ) -  { -  int quant = (int)args->quant; -  string format = lower_case(args->format || "gif"); -  string dither = args->dither; -  Image.Colortable ct; -  Image.Color.Color bgcolor; -  Image.Image alpha; -  int true_alpha; -  -  if( args->fs || dither == "fs" ) -  dither = "floyd_steinberg"; -  -  if( dither == "random" ) -  dither = "random_dither"; -  -  if( format == "jpg" ) -  format = "jpeg"; -  -  if( dither ) -  dither = replace( dither, "-", "_" ); -  -  if(mappingp(reply)) -  { -  alpha = reply->alpha; -  reply = reply->img; -  } -  -  if( args["true-alpha"] ) -  true_alpha = 1; -  -  if( args["background"] || args["background-color"]) -  bgcolor = Image.Color( args["background"]||args["background-color"] ); -  -  if( args["opaque-value"] ) -  { -  if( !bgcolor ) -  true_alpha = 1; -  int ov = (int)(((float)args["opaque-value"])*2.55); -  if( ov < 0 ) -  ov = 0; -  else if( ov > 255 ) -  ov = 255; -  if( alpha ) -  alpha *= ov; -  else -  alpha = Image.Image( reply->xsize(), reply->ysize(), ov,ov,ov ); -  } -  -  if( args->gamma ) -  reply = reply->gamma( (float)args->gamma ); -  -  -  if( bgcolor && alpha && !true_alpha ) -  { -  reply = Image.Image( reply->xsize(), -  reply->ysize(), bgcolor ) -  ->paste_mask( reply, alpha ); -  alpha = alpha->threshold( 4 ); -  } -  -  int x0, y0, x1=reply->xsize(), y1=reply->ysize(), xc, yc; -  if( args["x-offset"] || args["xoffset"] ) -  x0 = (int)(args["x-offset"]||args["xoffset"]); -  if( args["y-offset"] || args["yoffset"] ) -  y0 = (int)(args["y-offset"]||args["yoffset"]); -  if( args["width"] || args["x-size"] ) -  x1 = (int)(args["x-size"]||args["width"]); -  if( args["height"] || args["y-size"] ) -  y1 = (int)(args["y-size"]||args["height"]); -  -  array xguides, yguides; -  function sort_guides = lambda() -  { -  xguides = ({}); yguides = ({}); -  if( guides ) -  { -  foreach( guides, object g ) -  if( g->pos > 0 ) -  if( g->vertical ) -  { -  if( g->pos < reply->xsize() ) -  xguides += ({ g->pos }); -  } -  else -  if( g->pos < reply->ysize() ) -  yguides += ({ g->pos }); -  sort( xguides ); sort( yguides ); -  } -  }; -  -  if( args->crop ) -  { -  int gx=1, gy=1, gx2, gy2; -  if( sscanf( args["guides-index"]||"", "%d,%d", gx, gy ) == 2 ) -  { -  gx2 = gx+1; -  gy2 = gy+1; -  sscanf( args["guides-index"]||"", "%d,%d-%d,%d", gx, gy, gx2, gy2 ); -  } -  /* No, I did not forget the break statements. */ -  -  switch( args->crop ) -  { -  case "guides-cross": -  sort_guides(); -  if( sizeof(xguides) && sizeof(yguides) ) -  { -  xc = xguides[ min(sizeof(xguides),gx) - 1 ]; -  yc = yguides[ min(sizeof(yguides),gy) - 1 ]; -  break; -  } -  guides=0; -  case "guides-region": -  sort_guides(); -  if( (sizeof(xguides)>1) && (sizeof(yguides)>1) ) -  { -  gx = min(sizeof(xguides)-1,gx) - 1; -  gy = min(sizeof(yguides)-1,gy) - 1; -  gx2 = min(sizeof(xguides),gx2) - 1; -  gy2 = min(sizeof(yguides),gy2) - 1; -  -  x0 = xguides[gx]; x1 = xguides[gx2] - x0; -  y0 = yguides[gy]; y1 = yguides[gy2] - y0; -  break; -  } -  default: -  if( sscanf( args->crop, "%d,%d-%d,%d", x0, y0, x1, y1 ) == 4) -  { -  x1 -= x0; -  y1 -= y0; -  } -  break; -  case "auto": -  [ x0, y0, x1, y1 ] = reply->find_autocrop(); -  x1 = x1 - x0 + 1; -  y1 = y1 - y0 + 1; -  } -  } -  sort_guides = 0; // To avoid garbage. -  - #define SCALEI 1 - #define SCALEF 2 - #define SCALEA 4 - #define CROP 8 -  -  function do_scale_and_crop = lambda ( int x0, int y0, -  int x1, int y1, -  int|float w, int|float h, -  int type ) -  { -  if( (type & CROP) && x1 && y1 -  && ((x1 != reply->xsize()) || (y1 != reply->ysize()) -  || x0 || y0 ) ) -  { -  reply = reply->copy( x0, y0, x0+x1-1, y0+y1-1, -  (bgcolor?bgcolor->rgb():0) ); -  if( alpha ) -  alpha = alpha->copy( x0, y0, x0+x1-1,y0+y1-1, Image.Color.black); -  } -  -  if( type & SCALEI ) -  { -  if( xc || yc ) -  { -  if( h && !w ) -  w = (reply->xsize() * h) / reply->ysize(); -  -  if( w && !h ) -  h = (reply->ysize() * w) / reply->xsize(); -  -  x0 = max( xc - w/2, 0 ); -  y0 = max( yc - h/2, 0 ); -  -  x1 = w; y1 = h; -  if( x0 + x1 > reply->xsize() ) -  { -  x0 = reply->xsize()-w; -  if( x0 < 0 ) -  { -  x0 = 0; -  x1 = reply->xsize(); -  } -  } -  if( y0 + y1 > reply->ysize() ) -  { -  y0 = reply->ysize()-h; -  if( y0 < 0 ) -  { -  y0 = 0; -  y1 = reply->ysize(); -  } -  } -  reply = reply->copy( x0, y0, x0+x1-1, y0+y1-1, -  (bgcolor?bgcolor->rgb():0) ); -  -  if( alpha ) -  alpha = alpha->copy( x0, y0, x0+x1-1,y0+y1-1, Image.Color.black); -  } -  } -  -  -  if( (type & SCALEF) && (w != 1.0) ) -  { -  reply = reply->scale( w ); -  if( alpha ) -  alpha = alpha->scale( w ); -  } -  else if( (type & SCALEA) && -  ((reply->xsize() != w) || (reply->ysize() != h)) ) -  { -  reply = reply->scale( w,h ); -  if( alpha ) -  alpha = alpha->scale( w,h ); -  } -  else if( (type & SCALEI) && -  ((reply->xsize() != w) || (reply->ysize() != h)) ) -  { -  if( w && h ) -  { -  if( (w * (float)reply->ysize()) < (h * (float)reply->xsize()) ) -  h = 0; -  else -  w = 0; -  } -  w = min( w, reply->xsize() ); -  h = min( h, reply->ysize() ); -  reply = reply->scale( w,h ); -  if( alpha ) -  alpha = alpha->scale( w,h ); -  } -  }; -  -  if( sizeof((string) (args->scale || "")) ) -  { -  int x, y; -  if( sscanf( args->scale, "%d,%d", x, y ) == 2) -  do_scale_and_crop( x0, y0, x1, y1, x, y, SCALEA|CROP ); -  else if( (float)args->scale < 3.0) -  do_scale_and_crop( x0, y0, x1, y1, -  ((float)args->scale), ((float)args->scale), -  SCALEF|CROP ); -  } -  else -  if( sizeof( (string) (args->maxwidth || args->maxheight || -  args["max-width"] || args["max-height"] || "")) ) -  { -  int x = (int)args->maxwidth|| (int)args["max-width"]; -  int y = (int)args->maxheight||(int)args["max-height"]; -  do_scale_and_crop( x0, y0, x1, y1, x, y, SCALEI|CROP ); -  } -  else -  do_scale_and_crop( x0, y0, x1, y1, 0, 0, CROP ); -  do_scale_and_crop = 0; // To avoid garbage. -  -  if( args["span-width"] || args["span-height"] ) -  { -  int width = (int)args["span-width"]; -  int height = (int)args["span-height"]; -  -  if( (width && reply->xsize() > width) || -  (height && reply->ysize() > height) ) -  { -  if( (width && height && (reply->xsize() / (float)width > -  reply->ysize() / (float)height)) || -  !height ) -  { -  reply = reply->scale( width, 0 ); -  if( alpha ) -  alpha = alpha->scale( width, 0 ); -  } -  else if( height ) -  { -  reply = reply->scale( 0, height ); -  if( alpha ) -  alpha = alpha->scale( 0, height ); -  } -  } -  -  int x1,x2,y1,y2; -  if( width ) -  { -  x1 = -((width - reply->xsize()) / 2); -  x2 = x1 + width - 1; -  } -  if( height ) -  { -  y1 = -((height - reply->ysize()) / 2); -  y2 = y1 + height - 1; -  } -  -  if( width && height ) -  { -  reply = reply->copy(x1,y1,x2,y2,(bgcolor?bgcolor->rgb():0)); -  if( alpha ) alpha = alpha->copy(x1,y1,x2,y2); -  } -  else if( width ) -  { -  reply = reply->copy(x1,0,x2,reply->ysize(),(bgcolor?bgcolor->rgb():0)); -  if ( alpha ) alpha = alpha->copy(x1,0,x2,alpha->ysize()); -  } -  else -  { -  reply = reply->copy(0,y1,reply->xsize(),y2,(bgcolor?bgcolor->rgb():0)); -  if( alpha ) alpha = alpha->copy(0,y1,alpha->xsize(),y2); -  } -  } -  -  if( args["rotate-cw"] || args["rotate-ccw"]) -  { -  float degree = (float)(args["rotate-cw"] || args["rotate-ccw"]); -  switch( args["rotate-unit"] && args["rotate-unit"][0..0] ) -  { -  case "r": degree = (degree / (2*3.1415)) * 360; break; -  case "d": break; -  case "n": degree = (degree / 400) * 360; break; -  case "p": degree = (degree / 1.0) * 360; break; -  } -  if( args["rotate-cw"] ) -  degree = -degree; -  if(!alpha) -  alpha = reply->copy()->clear(255,255,255); -  reply = reply->rotate_expand( degree ); -  alpha = alpha->rotate( degree, 0,0,0 ); -  } -  -  -  if( args["mirror-x"] ) -  { -  if( alpha ) -  alpha = alpha->mirrorx(); -  reply = reply->mirrorx(); -  } -  -  if( args["mirror-y"] ) -  { -  if( alpha ) -  alpha = alpha->mirrory(); -  reply = reply->mirrory(); -  } -  -  if( bgcolor && alpha && !true_alpha ) -  { -  reply = Image.Image( reply->xsize(), -  reply->ysize(), bgcolor ) -  ->paste_mask( reply, alpha ); -  } -  -  if( args["cs-rgb-hsv"] )reply = reply->rgb_to_hsv(); -  if( args["cs-grey"] ) reply = reply->grey(); -  if( args["cs-invert"] ) reply = reply->invert(); -  if( args["cs-hsv-rgb"] )reply = reply->hsv_to_rgb(); -  -  if( !true_alpha && alpha ) -  alpha = alpha->threshold( 4 ); -  -  if( quant || (format=="gif") ) -  { -  int ncols = quant; -  if( format=="gif" ) { -  ncols = ncols||id->misc->defquant||32; -  if( ncols > 254 ) -  ncols = 254; -  } -  ct = Image.Colortable( reply, ncols ); -  if( dither ) -  { -  if( dither == "random" ) -  dither = "random_grey"; -  if( ct[ dither ] ) -  ct[ dither ](); -  else -  ct->ordered(); -  -  } -  } -  -  mapping enc_args = ([]); -  if( ct ) -  enc_args->colortable = ct; -  -  if( alpha ) -  enc_args->alpha = alpha; -  -  foreach( glob( "*-*", indices(args)), string n ) -  if(sscanf(n, "%*[^-]-%s", string opt ) == 2) -  if( opt != "alpha" ) -  enc_args[opt] = (int)args[n]; -  -  switch(format) -  { - #if constant(Image.WebP) && constant(Image.WebP.encode) -  case "webp": -  // Only mixed case module -  data = Image.WebP.encode( reply, enc_args ); -  break; - #endif -  case "wbf": -  format = "wbmp"; -  case "wbmp": -  Image.Colortable bw=Image.Colortable( ({ ({ 0,0,0 }), -  ({ 255,255,255 }) }) ); -  bw->floyd_steinberg(); -  data = Image.WBF.encode( bw->map( reply ), enc_args ); -  break; -  case "gif": - #if constant(Image.GIF) && constant(Image.GIF.encode) -  if( alpha && true_alpha ) -  { -  Image.Colortable bw=Image.Colortable( ({ ({ 0,0,0 }), -  ({ 255,255,255 }) }) ); -  bw->floyd_steinberg(); -  alpha = bw->map( alpha ); -  } -  if( catch { -  if( alpha ) -  data = Image.GIF.encode_trans( reply, ct, alpha ); -  else -  data = Image.GIF.encode( reply, ct ); -  }) -  data = Image.GIF.encode( reply ); -  break; -  - #else -  // Fall-through when there is no GIF encoder available -- -  // use PNG with a colortable instead. -  format = "png"; - #endif -  -  case "png": -  if( ct ) enc_args->palette = ct; -  m_delete( enc_args, "colortable" ); -  if( !(args["png-use-alpha"] || args["true-alpha"]) ) -  m_delete( enc_args, "alpha" ); -  else if( enc_args->alpha ) -  // PNG encoder doesn't handle alpha and palette simultaneously -  // which is rather sad, since that's the only thing 100% supported -  // by all common browsers. -  m_delete( enc_args, "palette"); -  else -  m_delete( enc_args, "alpha" ); -  -  default: -  if(!Image[upper_case( format )] -  || !Image[upper_case( format )]->encode ) -  error("Image format "+format+" not supported\n"); -  data = Image[upper_case( format )]->encode( reply, enc_args ); -  } -  -  meta = -  ([ -  "xsize":reply->xsize(), -  "ysize":reply->ysize(), -  "type":(format == "wbmp" ? "image/vnd.wap.wbmp" : "image/"+format ), -  ]); -  } -  else if( mappingp(reply) ) -  { -  // This could be an error from get_file() -  if(reply->error) -  return reply; -  meta = reply->meta; -  data = reply->data; -  if( !meta || !data ) -  error("Invalid reply mapping.\n" -  "Expected ([ \"meta\": ([metadata]), \"data\":\"data\" ])\n" -  "Got %O\n", reply); -  } - #ifdef ARG_CACHE_DEBUG -  werror("draw %O done\n", name ); - #endif -  // Avoid throwing and error if the same image is rendered twice. -  mixed err = catch(store_data( name, data, meta )); -  if (objectp (err) && err->is_RXML_Backtrace && RXML_CONTEXT) { -  throw (err); -  } - #ifdef ARG_CACHE_DEBUG -  if (err) { -  werror("store_data failed with:\n" -  "%s\n", describe_backtrace(err)); -  } - #endif -  } -  -  protected void store_data( string id, string data, mapping meta ) -  { -  if(!stringp(data)) return; - #ifdef ARG_CACHE_DEBUG -  werror("store %O (%d bytes)\n", id, strlen(data) ); - #endif -  // Since 134217728 (2^27) is the largest number we can give as argument to -  // the SQL-function "SPACE()", we cannot store images larger than 2^27 bytes -  // even though the size of the data-column allows 4294967295 bytes. -  if (sizeof(data) > 134217728) { // 134217728 = 2^27 -  RXML.run_error("Generated image data (%f MB) exceeds max limit of " -  "128 MB.\n", (float) sizeof(data) / 1024 / 1024 ); -  } -  meta_cache_insert( id, meta ); -  string meta_data = encode_value( meta ); - #ifdef ARG_CACHE_DEBUG -  werror("Replacing entry for %O\n", id ); - #endif -  if (sizeof(data) <= 8*1024*1024) { -  // Should fit in the 16 MB query limit without problem. -  // Albeit it might trigger a slow query entry for large -  // entries. -  QUERY("REPLACE INTO " + name + -  " (id,size,atime,meta,data) VALUES" -  " (%s,%d,UNIX_TIMESTAMP()," MYSQL__BINARY "%s," MYSQL__BINARY "%s)", -  id, strlen(data)+strlen(meta_data), meta_data, data ); -  } else { -  // We need to perform multiple queries. - #ifdef ARG_CACHE_DEBUG -  werror("Writing %d bytes of padding for %s.\n", sizeof(data), id); - #endif -  array(string) a = data/(8.0*1024*1024); -  // NB: We clear the meta field to ensure that the entry -  // is invalid while we perform the insert. -  QUERY("REPLACE INTO " + name + -  " (id,size,atime,meta,data) VALUES" -  " (%s,%d,UNIX_TIMESTAMP(),'',SPACE(%d))", -  id, strlen(data)+strlen(meta_data), sizeof(data)); -  int pos; -  foreach(a, string frag) { - #ifdef ARG_CACHE_DEBUG -  werror("Writing fragment at position %d for %s.\n", pos, id); - #endif -  QUERY("UPDATE " + name + -  " SET data = INSERT(data, %d, %d, "MYSQL__BINARY "%s)" -  " WHERE id = %s", -  pos+1, sizeof(frag), frag, id); -  pos += sizeof(frag); -  } -  /* Set the meta data field to a valid value to enable the entry. */ - #ifdef ARG_CACHE_DEBUG -  werror("Writing metadata for %s.\n", id); - #endif -  QUERY("UPDATE " + name + -  " SET meta = " MYSQL__BINARY "%s" -  " WHERE id = %s", -  meta_data, id); -  } - #ifdef ARG_CACHE_DEBUG -  array(mapping(string:string)) q = -  QUERY("SELECT meta, data FROM " + name + -  " WHERE id = %s", id); -  if (!q || sizeof(q) != 1) { -  werror("Unexpected result size: %d\n", -  q && sizeof(q)); -  } else { -  if (q[0]->meta != meta_data) { -  werror("Meta data differs: %O != %O\n", -  meta_data, q[0]->meta); -  } -  if (q[0]->data != data) { -  string d = q[0]->data; -  int i; -  int cnt; -  for (i = 0; i < sizeof(data); i++) { -  if (data[i] == d[i]) continue; -  werror("Data differs at offset %d: %d != %d\n", -  i, data[i], d[i]); -  if (cnt++ > 10) break; -  } -  } -  } - #endif -  } -  -  protected mapping restore_meta( string id, RequestID rid ) -  { -  if( array item = meta_cache[ id ] ) -  { -  item[ 1 ] = time(1); // Update cached atime. -  return item[ 0 ]; -  } -  - #ifdef ARG_CACHE_DEBUG -  werror("restore meta %O\n", id ); - #endif -  array(mapping(string:string)) q = -  QUERY("SELECT meta, UNIX_TIMESTAMP()-CAST(atime AS SIGNED) AS atime_diff " -  "FROM "+name+" WHERE id=%s", id ); -  -  string s; -  if(!sizeof(q) || !strlen(s = q[0]->meta)) -  return 0; -  -  mapping m; -  if (catch (m = decode_value (s))) -  { -  report_error( "Corrupt data in cache-entry "+id+".\n" ); -  QUERY( "DELETE FROM "+name+" WHERE id=%s", id); -  return 0; -  } -  -  // Update atime only if older than threshold. -  if((int)q[0]->atime_diff > ATIME_THRESHOLD) { -  QUERY("UPDATE LOW_PRIORITY "+name+" " -  " SET atime = UNIX_TIMESTAMP() " -  " WHERE id = %s ", id); -  } -  return meta_cache_insert( id, m ); -  } -  -  protected void sync_meta() -  { -  mapping tmp = meta_cache; -  meta_cache = ([]); -  // Sync cached atimes. -  foreach(tmp; string id; array value) { -  if (value[1]) -  QUERY("UPDATE "+name+" SET atime=%d WHERE id=%s", -  value[1], id); -  } -  } -  -  void flush(int|void age) -  //! Flush the cache. If an age (an integer as returned by -  //! @[time()]) is provided, only images with their latest access before -  //! that time are flushed. -  { -  int num; - #if defined(DEBUG) || defined(IMG_CACHE_DEBUG) -  int t = gethrtime(); - #endif -  sync_meta(); -  uid_cache = ([]); -  rst_cache = ([]); -  if( !age ) -  { -  QUERY( "DELETE FROM "+name ); - #if defined(DEBUG) || defined(IMG_CACHE_DEBUG) -  int msec = (gethrtime() - t) / 1000; -  report_debug("Image cache %s emptied (%dms).\n", name, msec); - #endif -  num = -1; -  return; -  } -  -  array(string) ids = -  QUERY( "SELECT id FROM "+name+" WHERE atime < "+age)->id; -  -  num = sizeof( ids ); -  -  int q; -  while(q<sizeof(ids)) { -  string list = map(ids[q..q+99], get_db()->quote) * "','"; -  q+=100; -  QUERY( "DELETE LOW_PRIORITY FROM "+name+" WHERE id in ('"+list+"')" ); -  } -  - #if 0 -  // Disabled. This can take a significant amount of time to run, -  // and we really can't afford an unresponsive image cache - it can -  // easily hang all handler threads. Besides, it's doubtful if this -  // is of any use since the space for the deleted records probably -  // will get reused soon enough anyway. /mast -  if( num ) -  catch -  { -  // Old versions of Mysql lacks OPTIMIZE. Not that we support -  // them, really, but it might be nice not to throw an error, at -  // least. - #if defined(DEBUG) || defined(IMG_CACHE_DEBUG) -  report_debug("Optimizing database ... ", name); - #endif -  QUERY( "OPTIMIZE TABLE "+name ); -  }; - #endif -  - #if defined(DEBUG) || defined(IMG_CACHE_DEBUG) -  int msec = (gethrtime() - t) / 1000; -  if (num || (msec > 500)) { -  report_debug("Image cache %s cleaned: %s removed (%dms)\n", -  name, -  (num == -1 ? "all" : num ? (string) num : "none"), msec); -  } - #endif -  } -  -  array(int) status(int|void age) -  //! Return the total number of images in the cache, their cumulative -  //! sizes in bytes and, if an age time_t was supplied, the number of -  //! images that has not been accessed after that time is returned -  //! (see <ref>flush()</ref>). (Three integers are returned -  //! regardless of whether an age parameter was given.) -  { -  int imgs=0, size=0, aged=0; -  array(mapping(string:string)) q; -  -  q=QUERY("SHOW TABLE STATUS"); -  foreach(q, mapping qq) -  if(has_prefix(qq->Name, name)) { -  imgs = (int)qq->Rows; -  size += (int)qq->Data_length; -  } -  -  if(age) { -  q=QUERY("select SUM(1) as num from "+name+" where atime < "+age); -  aged = (int)q[0]->num; -  } -  return ({ imgs, size, aged }); -  } -  -  protected mapping(string:mapping) rst_cache = ([ ]); -  protected mapping(string:string) uid_cache = ([ ]); -  -  protected mapping restore( string id, RequestID rid ) -  { -  array q; -  string uid; -  if( zero_type(uid = uid_cache[id]) ) -  { -  q = QUERY( "SELECT uid FROM "+name+" WHERE id=%s",id); -  if( sizeof(q) ) -  uid = q[0]->uid; -  else -  uid = 0; -  uid_cache[id] = uid; -  } -  -  if( uid && strlen(uid) ) -  { -  User u; -  if( !(u=rid->conf->authenticate(rid)) || (u->name() != uid ) ) -  return rid->conf->authenticate_throw(rid, "User"); -  } -  -  if( rst_cache[ id ] ) { -  rid->cache_status["icacheram"] = 1; -  return rst_cache[ id ] + ([]); -  } -  - #ifdef ARG_CACHE_DEBUG -  werror("restore %O\n", id ); - #endif -  q = QUERY( "SELECT meta,atime,data FROM "+name+" WHERE id=%s",id); -  if( sizeof(q) ) -  { -  if( sizeof(q[0]->data) ) -  { -  // Case 1: We have cache entry and image. -  string f = q[0]->data; -  mapping m; -  if (mixed err = catch( m = decode_value( q[0]->meta ) )) -  report_debug ("Failed to decode meta mapping for id %O in %s: %s", -  id, name, describe_error (err)); -  if( !m ) return 0; -  -  rid->cache_status["icachedisk"] = 1; -  m = Roxen.http_string_answer( f, m->type||("image/gif") ); -  -  if( strlen( f ) > 6000 ) -  return m; -  rst_cache[ id ] = m; -  if( sizeof( rst_cache ) > 100 ) -  rst_cache = ([ id : m ]); -  return rst_cache[ id ] + ([]); -  } -  // Case 2: We have cache entry, but no data. -  return 0; -  } -  else -  { -  // Case 3: No cache entry. Create one -  User u = rid->conf->authenticate(rid); -  string uid = ""; -  if( u ) uid = u->name(); -  // Might have been insterted from elsewhere. -  QUERY("REPLACE INTO "+name+ -  " (id,uid,atime) VALUES (%s,%s,UNIX_TIMESTAMP())", -  id, uid ); -  } -  -  return 0; -  } -  -  -  string data( array|string|mapping args, RequestID id, int|void nodraw, -  int|void timeout ) -  //! Returns the actual raw image data of the image rendered from the -  //! @[args] instructions. -  //! -  //! A non-zero @[nodraw] parameter means an image not already in the -  //! cache will not be rendered on the fly, but instead return zero. -  { -  mapping res = http_file_answer( args, id, nodraw, timeout ); -  return res && res->data; -  } -  -  mapping http_file_answer( array|string|mapping data, -  RequestID id, -  int|void nodraw, int|void timeout ) -  //! Returns a @[result mapping] like one generated by -  //! @[Roxen.http_file_answer()] but for the image file -  //! rendered from the `data' instructions. -  //! -  //! Like @[metadata], a non-zero @[nodraw]parameter means an -  //! image not already in the cache will not be rendered on the fly, -  //! but instead zero will be returned (this will be seen as a 'File -  //! not found' error) -  { -  current_configuration->set(id->conf); -  string na = store( data, id, timeout ); -  mixed res; - #ifdef ARG_CACHE_DEBUG -  werror("data: %O id: %O\n", na, id ); - #endif -  if(! (res=restore( na,id )) ) -  { -  mixed err; -  if (nodraw || (err = catch { -  if (mapping res = draw( na, id )) -  return res; -  })) { - #ifdef ARG_CACHE_DEBUG -  werror("draw() failed with error: %s\n", -  describe_backtrace(err)); - #endif -  if (objectp (err) && err->is_RXML_Backtrace) { -  if (RXML_CONTEXT) { - #ifdef ARG_CACHE_DEBUG -  werror("Rethrowing error...\n"); - #endif -  throw (err); -  } -  // If we get an rxml error and there's no rxml context then -  // we're called from a direct request to the image cache. -  // The error ought to have been reported in the page that -  // generated the link to the image cache, but since it's too -  // late for that now, we just log it as a (brief) server -  // error with the referring page. -  string errmsg = "Error in " + name + " image generation: " + -  err->msg; -  if (sizeof (id->referer)) -  errmsg += " Referrer: " + id->referer[0]; -  report_error (errmsg + "\n"); -  return 0; -  } else if (arrayp(err) && sizeof(err) && stringp(err[0])) { -  if (sscanf(err[0], "Requesting unknown key %s\n", -  string message) == 1) -  { -  // File not found. -  report_debug("Requesting unknown key %s %O from %O\n", -  message, -  id->not_query, -  (sizeof(id->referer)?id->referer[0]:"unknown page")); -  return 0; -  } -  } - #ifdef ARG_CACHE_DEBUG -  werror("Rethrowing error...\n"); - #endif -  throw (err); -  } -  if( !(res = restore( na,id )) ) { -  report_error("Draw callback %O did not generate any data.\n" -  "na: %O\n" -  "id: %O\n", -  draw_function, na, id); -  return 0; -  } -  } -  res->stat = ({ 0, 0, 0, 900000000, 0, 0, 0, 0, 0 }); -  -  // Setting the cacheable flag is done in order to get headers sent which -  // cause the image to be cached in the client even when using https -  // sessions. -  // -  // NB: Raise it above INITIAL_CACHEABLE to force an Expires header. -  RAISE_CACHE(31557600); // A year. -  -  // With the new (5.0 and newer) arg-cache enabled by default we can -  // allow authenticated images in the protocol cache. At this point -  // http.pike will have cleared it so re-enable explicitly. -  PROTO_CACHE(); -  -  return res; -  } -  -  mapping metadata( array|string|mapping data, -  RequestID id, -  int|void nodraw, -  int|void timeout ) -  //! Returns a mapping of image metadata for an image generated from -  //! the data given (as sent to @[store()]). If a non-zero -  //! @[nodraw] parameter is given and the image was not in the cache, -  //! it will not be rendered on the fly to get the correct data. -  //! -  //! @param timeout -  //! The @[timeout] is sent unmodified to @[store()]. -  { -  string na = store( data, id, timeout ); -  mapping res; - #ifdef ARG_CACHE_DEBUG -  werror("meta %O\n", na ); - #endif -  if(! (res = restore_meta( na,id )) ) -  { -  if(nodraw) -  return 0; -  draw( na, id ); -  return restore_meta( na,id ); -  } -  return res; -  } -  -  mapping tomapp( mixed what ) -  { -  if( mappingp( what )) -  return what; -  return ([ "":what ]); -  } -  -  string store( array|string|mapping data, RequestID id, int|void timeout ) -  //! Store the data your draw callback expects to receive as its -  //! first argument(s). If the data is an array, the draw callback -  //! will be called like <pi>callback( @@data, id )</pi>. -  //! -  //! @param timeout -  //! Timeout for the entry in seconds from now. If @expr{UNDEFINED@}, -  //! the entry will not expire. Currently just passed along to -  //! the @[ArgCache]. -  { -  string ci, user; -  function update_args = lambda ( mapping a ) -  { -  if (!a->format) -  // Make implicit format choice explicit - #if constant(Image.GIF) && constant(Image.GIF.encode) -  a->format = "gif"; - #else -  a->format = "png"; - #endif -  if( get_admin_configuration() != id->conf && -  id->misc->authenticated_user && -  !id->misc->authenticated_user->is_transient ) -  // This entry is not actually used, it's only there to -  // generate a unique key. -  a["\0u"] = user = id->misc->authenticated_user->name(); -  }; -  -  if( mappingp( data ) ) -  { -  update_args( data ); -  ci = argcache->store( data, timeout ); -  } -  else if( arrayp( data ) ) -  { -  if( !mappingp( data[0] ) ) -  error("Expected mapping as the first element of the argument array\n"); -  update_args( data[0] ); -  ci = map( map( data, tomapp ), argcache->store, timeout )*"$"; -  } else -  ci = data; -  update_args = 0; // To avoid garbage. -  -  if( zero_type( uid_cache[ ci ] ) ) -  { -  uid_cache[ci] = user; -  // Make sure to only update the entry if it does not already -  // exists or has wrong uid. Allways updating the table will -  // casue mysql to lock the table and cause a potential gobal -  // ImageCache stall. -  string uid = user || ""; -  array q = QUERY("SELECT uid from "+name+" where id=%s", ci); -  if(!sizeof(q) || (sizeof(q) && q[0]->uid != uid)) { -  QUERY("INSERT INTO "+name+" " -  "(id,uid,atime) VALUES (%s,%s,UNIX_TIMESTAMP()) " -  "ON DUPLICATE KEY UPDATE uid=%s", -  ci, uid, uid); -  } -  } -  - #ifndef NO_ARG_CACHE_SB_REPLICATE -  if(id->misc->persistent_cache_crawler || id->misc->do_replicate_argcache) { -  // Force an update of rep_time for the requested arg cache id. -  foreach(ci/"$", string key) { - #if ARGCACHE_DEBUG -  werror("Request for id %O from prefetch crawler.\n", key); - #endif /* ARGCACHE_DEBUG */ -  argcache->refresh_arg(key); -  } -  } - #endif /* NO_ARG_CACHE_SB_REPLICATE */ -  return ci; -  } -  -  void set_draw_function( function to ) -  //! Set a new draw function. -  { -  draw_function = to; -  } -  -  protected void setup_tables() -  { -  if(catch(QUERY("SELECT data FROM "+name+" WHERE id=''"))) -  { -  werror("Creating image-cache tables for '"+name+"'\n"); -  catch(QUERY("DROP TABLE "+name)); -  -  // The old tables. This is only useful for people who have run -  // Roxen 2.2 from cvs before -  catch(QUERY("DROP TABLE "+name+"_data")); -  -  -  master()->resolv("DBManager.is_module_table") -  ( 0,"local",name,"Image cache for "+name); -  -  QUERY("CREATE TABLE "+name+" (" -  "id CHAR(64) NOT NULL PRIMARY KEY, " -  "size INT UNSIGNED NOT NULL DEFAULT 0, " -  "uid CHAR(32) NOT NULL DEFAULT '', " -  "atime INT UNSIGNED NOT NULL DEFAULT 0," -  "meta MEDIUMBLOB NOT NULL DEFAULT ''," -  "data LONGBLOB NOT NULL DEFAULT ''," -  "INDEX atime_id (atime, id)" -  ")" ); -  } -  -  // Inhibit backups of this table. -  master()->resolv("DBManager.inhibit_backups")("local", name); -  -  // Create index in old databases. Index is used when flushing old -  // entries. Column 'id' is included in index in order to avoid -  // reading data file. -  array(mapping(string:mixed)) res = QUERY("SHOW INDEX FROM " + name); -  if(search(res->Key_name, "atime_id") < 0) { -  report_debug("Updating " + name + " image cache: " -  "Adding index atime_id on %s... ", name); -  int start_time = gethrtime(); -  QUERY("CREATE INDEX atime_id ON " + name + " (atime, id)"); -  report_debug("complete. [%f s]\n", (gethrtime() - start_time)/1000000.0); -  report_debug("Updating " + name + " image cache: " -  "Dropping index atime on %s... ", name); -  start_time = gethrtime(); -  QUERY("DROP INDEX atime ON " + name); -  report_debug("complete. [%f s]\n", (gethrtime() - start_time)/1000000.0); -  } -  res = QUERY("SHOW COLUMNS FROM " + name + " WHERE Field = 'data'"); -  if (lower_case(res[0]->Type) != "longblob") { -  report_debug("Updating " + name + " image cache: " -  "Increasing maximum blob size..."); -  int start_time = gethrtime(); -  QUERY("ALTER TABLE " + name + -  " MODIFY COLUMN data LONGBLOB NOT NULL DEFAULT ''"); -  report_debug("complete. [%f s]\n", (gethrtime() - start_time)/1000000.0); -  } -  } -  -  Sql.Sql get_db() -  { -  return dbm_cached_get("local"); -  } -  -  protected void init_db( ) -  { -  catch(sync_meta()); -  setup_tables(); -  } -  -  void do_cleanup( ) -  { -  // Flushes may be costly in large sites (at least the OPTIMIZE TABLE -  // command) so schedule next run sometime after 04:30 the day after -  // tomorrow. -  // -  // Note: The OPTIMIZE TABLE step has been disabled. /mast -  int now = time(); -  mapping info = localtime(now); -  int wait = (int) (((24 - info->hour) + 24 + 4.5) % 24) * 3600 + random(500); -  background_run(wait, do_cleanup); -  -  // Remove items older than one week -  flush(now - 7 * 3600 * 24); -  } -  -  protected void create( string id, function draw_func ) -  //! Instantiate an image cache of your own, whose image files will -  //! be stored in a table `id' in the cache mysql database, -  //! -  //! The `draw_func' callback passed will be responsible for -  //! (re)generation of the images in the cache. Your draw callback -  //! may take any arguments you want, depending on the first argument -  //! you give the <ref>store()</ref> method, but its final argument -  //! will be the RequestID object. -  //! -  //! @note -  //! Use @[RXML.run_error] or @[RXML.parse_error] within the draw -  //! function to throw user level drawing errors, e.g. invalid or -  //! missing images or argument errors. If it's called within a -  //! graphics tag then the error is thrown directly and reported -  //! properly by the rxml evaluator. If it's called later, i.e. in a -  //! direct request to the image cache, then it is catched by the -  //! @[ImageCache] functions and reported in as good way as possible, -  //! i.e. currently briefly in the debug log. -  { -  name = id; -  draw_function = draw_func; -  init_db(); -  // Support that the 'local' database moves. -  master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db ); -  -  // Always remove entries that are older than one week. -  background_run( 10, do_cleanup ); -  } -  -  protected void destroy() -  { -  if (mixed err = catch(sync_meta())) { -  report_warning("Failed to sync cached atimes for "+name+"\n"); - #if 0 - #ifdef DEBUG -  master()->handle_error (err); - #endif - #endif -  } -  } - } -  -  - class ArgCache - //! Generic cache for storing away a persistent mapping of data to be - //! refetched later by a short string key. This being a cache, your - //! data may be thrown away at random when the cache is full. - { - #define GET_DB() \ -  Sql.Sql db = dbm_cached_get("local") - #undef QUERY - #define QUERY(X,Y...) db->query(X,Y) -  string name; -  - #define CACHE_SIZE 900 -  - #ifdef ARGCACHE_DEBUG - #define dwerror(ARGS...) werror(ARGS) - #else - #define dwerror(ARGS...) 0 - #endif -  -  //! Cache of the latest entries requested or stored. -  //! Limited to @[CACHE_SIZE] (currently @expr{900@}) entries. -  protected mapping(string|int:mixed) cache = ([ ]); -  -  //! Cache of cache-ids that have no expiration time. -  //! This cache is maintained in sync with @[cache]. -  //! Note that entries not in this cache may still have -  //! unlimited expiration time. -  protected mapping(string|int:int) no_expiry = ([ ]); -  -  protected void setup_table() -  { -  GET_DB(); -  -  // New style argument2 table. -  if(catch(QUERY("SELECT id FROM "+name+"2 LIMIT 0"))) -  { -  master()->resolv("DBManager.is_module_table") -  ( 0, "local", name+"2", -  "The argument cache, used to map between " -  "a unique string and an argument mapping" ); -  catch(QUERY("DROP TABLE "+name+"2" )); -  QUERY("CREATE TABLE "+name+"2 (" -  "id CHAR(32) PRIMARY KEY, " -  "ctime DATETIME NOT NULL, " -  "atime DATETIME NOT NULL, " -  "rep_time DATETIME NOT NULL, " -  "sync_time INT NULL, " -  "timeout INT NULL, " -  "contents MEDIUMBLOB NOT NULL, " -  " INDEX(timeout)," -  " INDEX(sync_time)" -  ")"); -  } -  -  // Inhibit backups of the arguments2 table. -  master()->resolv("DBManager.inhibit_backups") -  ("local", name + "2"); -  -  if (catch (QUERY ("SELECT rep_time FROM " + name + "2 LIMIT 0"))) -  { -  // Upgrade a table without rep_time. -  QUERY ("ALTER TABLE " + name + "2" -  " ADD rep_time DATETIME NOT NULL" -  " AFTER atime"); -  } -  -  if (catch (QUERY ("SELECT timeout FROM " + name + "2 LIMIT 0"))) -  { -  // Upgrade a table without timeout. -  QUERY ("ALTER TABLE " + name + "2 " -  " ADD timeout INT NULL " -  "AFTER rep_time"); -  QUERY ("ALTER TABLE " + name + "2 " -  " ADD INDEX(timeout)"); -  } -  -  if (catch (QUERY ("SELECT sync_time FROM " + name + "2 LIMIT 0"))) -  { -  // Upgrade a table without sync_time. -  QUERY ("ALTER TABLE " + name + "2" -  " ADD sync_time INT NULL" -  " AFTER rep_time"); -  QUERY ("ALTER TABLE " + name + "2 " -  " ADD INDEX(sync_time)"); -  } -  -  catch { -  array(mapping(string:mixed)) res = -  QUERY("DESCRIBE "+name+"2 contents"); -  -  if(res[0]->Type == "blob") { -  QUERY("ALTER TABLE "+name+"2 MODIFY contents MEDIUMBLOB NOT NULL"); -  werror("ArgCache: Extending \"contents\" field in table \"%s2\" from BLOB to MEDIUMBLOB.\n", name); -  } -  }; -  } -  -  protected void do_cleanup() -  { -  GET_DB(); -  QUERY("DELETE LOW_PRIORITY FROM " + name + "2 " -  " WHERE timeout IS NOT NULL " -  " AND timeout < %d", time()); -  } -  -  protected void cleanup_process( ) -  { -  // Flushes may be costly in large sites (since there's no index -  // on the timeout field) so schedule next run sometime after -  // 04:30 the day after tomorrow. -  int now = time(); -  mapping info = localtime(now); -  int wait = (int) ((24 - info->hour) + 24 + 4.5) * 3600 + random(500); -  background_run(wait, cleanup_process); -  -  do_cleanup(); -  } -  -  protected void init_db() -  { -  // Delay DBManager resolving to before the 'roxen' object is -  // compiled. -  cache = ([]); -  no_expiry = ([]); -  setup_table( ); -  -  // Cleanup exprired entries on start. -  background_run( 10, cleanup_process ); -  } -  -  protected void create( string _name ) -  { -  name = _name; -  init_db(); -  // Support that the 'local' database moves (not really nessesary, -  // but it won't hurt either) -  master()->resolv( "DBManager.add_dblist_changed_callback" )( init_db ); -  get_plugins(); -  } -  -  protected string read_encoded_args( string id, int dont_update_atime ) -  { -  GET_DB(); -  array res = QUERY("SELECT contents FROM "+name+"2 " -  " WHERE id = %s", id); -  if(!sizeof(res)) -  return 0; -  if (!dont_update_atime) -  QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET atime = NOW() " -  " WHERE id = %s", id); -  return res[0]->contents; -  } -  -  // Callback used in replicate.pike -  void create_key( string id, string encoded_args, int|void timeout ) -  { -  if (!zero_type(timeout) && (timeout < time(1))) return; // Expired. -  GET_DB(); -  array(mapping) rows = -  QUERY("SELECT id, contents, timeout, " -  "UNIX_TIMESTAMP() - UNIX_TIMESTAMP(atime) as atime_diff " -  "FROM "+name+"2 " -  "WHERE id = %s", id ); -  -  foreach( rows, mapping row ) -  if( row->contents != encoded_args ) { -  report_error("ArgCache.create_key(): Duplicate key found! " -  "Please report this to support@roxen.com:\n" -  " id: %O\n" -  " old data: %O\n" -  " new data: %O\n" -  " Updating local database with new value.\n", -  id, row->contents, encoded_args); -  -  // Remove the old entry (probably corrupt). No need to update -  // the database since the query below uses REPLACE INTO. -  rows = ({}); -  } -  -  if(sizeof(rows)) { -  // Update atime only if older than threshold. -  if((int)rows[0]->atime_diff > ATIME_THRESHOLD) { -  QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET atime = NOW() " -  " WHERE id = %s", id); -  } -  -  // Increase timeout when needed. -  if (rows[0]->timeout) { -  if (zero_type(timeout)) { -  // No timeout, i.e. infinite timeout. -  QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET timeout = NULL " -  " WHERE id = %s", id); -  } else if (timeout > (int)rows[0]->timeout) { -  QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET timeout = %d " -  " WHERE id = %s", timeout, id); -  } -  } -  return; -  } -  -  string timeout_sql = zero_type(timeout) ? "NULL" : (string)timeout; -  // Use REPLACE INTO to cope with entries created by other threads -  // as well as corrupted entries that should be overwritten. -  QUERY( "REPLACE INTO "+name+"2 " -  "(id, contents, ctime, atime, timeout) VALUES " -  "(%s, " MYSQL__BINARY "%s, NOW(), NOW(), "+timeout_sql+")", -  id, encoded_args ); -  -  dwerror("ArgCache: Create new key %O\n", id); -  -  (plugins->create_key-({0}))( id, encoded_args ); -  } -  -  protected array plugins; -  protected void get_plugins() -  { -  plugins = ({}); -  foreach( ({ "../local/arg_cache_plugins", "arg_cache_plugins" }), string d) -  if( file_stat( d ) ) -  foreach( glob("*.pike", get_dir( d )), string f ) -  { -  object plug = ((program)(d+"/"+f))(this_object()); -  if( !plug->disabled ) -  plugins += ({ plug }); -  } -  } -  -  protected mapping plugins_read_encoded_args( string id ) -  { -  mapping args; -  foreach( (plugins->read_encoded_args - ({0})), function(string:mapping) f ) -  if( args = f( id ) ) -  return args; -  return 0; -  } -  -  string store( mapping args, int|void timeout ) -  //! Store a mapping (of purely encode_value:able data) in the -  //! argument cache. The string returned is your key to retrieve the -  //! data later. -  //! -  //! @param timeout -  //! Timeout for the entry in seconds from now. If @expr{UNDEFINED@}, -  //! the entry will not expire. -  { -  if (!zero_type(timeout)) timeout += time(1); -  string encoded_args = encode_value_canonic( args ); -  string id = Gmp.mpz(Crypto.SHA1.hash(encoded_args), 256)->digits(36); -  if( cache[ id ] ) { -  if (!no_expiry[id]) { -  // The cache id may have a timeout. -  GET_DB(); -  if (zero_type(timeout)) { -  // No timeout now, but there may have been one earlier. -  QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET timeout = NULL " -  " WHERE id = %s " -  " AND timeout IS NOT NULL", id); -  no_expiry[id] = 1; -  } else { -  // Attempt to bump the timeout. -  QUERY("UPDATE LOW_PRIORITY "+name+"2 " -  " SET timeout = %d " -  " WHERE id = %s " -  " AND timeout IS NOT NULL " -  " AND timeout < %d", -  timeout, id, timeout); -  } -  } -  return id; -  } -  create_key(id, encoded_args, timeout); -  if( sizeof( cache ) >= CACHE_SIZE ) { -  cache = ([]); -  no_expiry = ([]); -  } -  if( !cache[ id ] ) { -  cache[ id ] = args + ([]); -  } -  if (zero_type(timeout)) { -  no_expiry[ id ] = 1; -  } -  return id; -  } -  -  -  mapping lookup( string id ) -  //! Recall a mapping stored in the cache. -  { -  if( cache[id] ) -  return cache[id] + ([]); -  string encoded_args = (read_encoded_args(id, 0) || -  plugins_read_encoded_args(id)); -  if(!encoded_args) { -  error("Requesting unknown key (not found in db)\n"); -  } -  mapping args = decode_value(encoded_args); -  if( sizeof( cache ) >= CACHE_SIZE ) { -  // Yowza! Garbing bulldoze style. /mast -  cache = ([]); -  no_expiry = ([]); -  } -  cache[id] = args + ([]); -  return args; -  } -  -  void delete( string id ) -  //! Remove the data element stored under the key @[id]. -  { -  GET_DB(); -  (plugins->delete-({0}))( id ); -  m_delete( cache, id ); -  -  QUERY( "DELETE FROM "+name+"2 WHERE id = %s", id ); -  } -  -  int key_exists( string id ) -  //! Does the key @[id] exist in the cache? Returns 1 if it does, 0 -  //! if it was not present. -  { -  if( cache[id] ) return 1; -  if (read_encoded_args(id, 0) || plugins_read_encoded_args(id)) return 1; -  return 0; -  } -  - #define SECRET_TAG "££" -  -  int write_dump(Stdio.File file, int from_time) -  //! Dumps all entries that have been @[refresh_arg]'ed at or after -  //! @[from_time] to @[file]. All existing entries are dumped if -  //! @[from_time] is zero. -  //! -  //! @returns -  //! Returns 0 if writing failed, -1 if there was no new entries, 1 -  //! otherwise. -  //! -  //! @note -  //! Entries added during the execution of this function might or -  //! might not be included in the dump. -  { -  constant FETCH_ROWS = 10000; -  int entry_count = 0; -  -  // The server does only need to use file based argcache -  // replication if the server don't participate in a replicate -  // setup with a shared database. -  if( !has_value((plugins->is_functional-({0}))(), 1) ) -  { -  GET_DB(); -  int cursor; -  array(string) ids; -  do { -  // Note: No lock is held, so rows might be added between the -  // SELECTs here. That can however only cause a slight overlap -  // between the LIMIT windows since rows are only added and -  // never removed, and read_dump doesn't mind the occasional -  // duplicate entry. -  // -  // A lock will be necessary here if a garb is added, though. -  -  array(mapping(string:string)) entries; -  if(from_time) -  // Only replicate entries accessed during the prefetch crawling. -  entries = -  QUERY( "SELECT id, timeout from "+name+"2 " -  " WHERE rep_time >= FROM_UNIXTIME(%d) " -  " LIMIT %d, %d", from_time, cursor, FETCH_ROWS); -  else -  // Make sure _every_ entry is replicated when a dump is created. -  entries = -  QUERY( "SELECT id, timeout from "+name+"2 " -  " LIMIT %d, %d", cursor, FETCH_ROWS); -  -  ids = entries->id; -  array(string) timeouts = entries->timeout; -  cursor += FETCH_ROWS; -  -  foreach(ids; int i; string id) { -  dwerror("ArgCache.write_dump(): %O\n", id); -  -  string encoded_args; -  if (mapping args = cache[id]) -  encoded_args = encode_value_canonic (args); -  else { -  encoded_args = read_encoded_args (id, 1); -  if (!encoded_args) error ("ArgCache entry %O disappeared.\n", id); -  } -  -  string s; -  if (timeouts[i]) { -  int timeout = (int)timeouts[i]; -  if (timeout < time(1)) { -  // Expired entry. Don't replicate. -  continue; -  } -  s = -  MIME.encode_base64(encode_value(({ id, encoded_args, timeout })), -  1)+"\n"; -  } else { -  // No timeout. Backward-compatible format. -  s = -  MIME.encode_base64(encode_value(({ id, encoded_args })), -  1)+"\n"; -  } -  if(sizeof(s) != file->write(s)) -  return 0; -  entry_count++; -  } -  } while(sizeof(ids) == FETCH_ROWS); -  } -  if (file->write("EOF\n") != 4) -  return 0; -  return entry_count ? 1 : -1; -  } -  -  string read_dump (Stdio.FILE file) -  // Returns an error message if there was a parse error, 0 otherwise. -  { -  string secret = file->gets(); -  // Check if no secret is present -> newstyle package. -  if(!secret || !has_prefix(secret, SECRET_TAG)) -  // New pakage found, restore input stream. -  file->ungets(secret); -  -  string s; -  while(s = file->gets()) -  { -  if(s == "EOF") -  return 0; -  array a; -  if(mixed err = catch { -  a = decode_value(MIME.decode_base64(s)); -  }) return "Decode failed for argcache record: " + describe_error (err); -  -  if(sizeof(a) == 4) { -  // Old style argcache dump. -  dwerror("ArgCache.read_dump(): value_id: %O, index_id: %O.\n", a[0], a[2]); -  if (a[2] == -1) -  // The old write_dump didn't filter out entries with NULL -  // index_id when from_time was zero, so we ignore them here -  // instead. -  dwerror ("ArgCache.read_dump(): entry ignored.\n"); -  else { -  array v = decode_value(a[1]), i = decode_value(a[3]); - #if 0 -  dwerror ("ArgCache.read_dump(): values: %O, indices: %O\n", v, i); - #endif -  store(mkmapping(i, v)); -  } -  } else if ((sizeof(a) == 2) || (sizeof(a) == 3)) { -  // New style argcache dump, possibly with timeout. -  dwerror("ArgCache.read_dump(): %O\n", a[0]); -  create_key(@a); -  } else -  return "Decode failed for argcache record (wrong size on key array)\n"; -  } -  if(s != "EOF") -  return "Missing data in argcache file\n"; -  return 0; -  } -  -  void refresh_arg(string id) -  //! Indicate that the entry @[id] needs to be included in the next -  //! @[write_dump]. @[id] must be an existing entry. -  { -  GET_DB(); -  QUERY("UPDATE "+name+"2 SET rep_time=NOW() WHERE id = %s", id); -  } - } -  - mapping(string:Charset.Decoder) cached_decoders = ([]); - string decode_charset( string charset, string data ) - { -  // FIXME: This code is probably not thread-safe! -  if( charset == "iso-8859-1" ) return data; -  if( !cached_decoders[ charset ] ) -  cached_decoders[ charset ] = Charset.decoder( charset ); -  data = cached_decoders[ charset ]->feed( data )->drain(); -  cached_decoders[ charset ]->clear(); -  return data; - } -  - //! Check if a cache key has been marked invalid (aka stale). - int(0..1) invalidp(CacheKey key) - { -  catch { -  return !key || (key->invalidp && key->invalidp()); -  }; -  return !key; - } -  - //! Invalidate (mark as stale) a cache key. - void invalidate(CacheKey key) - { -  if (invalidp(key)) return; -  catch { -  if (key->invalidate) { -  key->invalidate(); -  return; -  } -  }; -  if (key) destruct(key); - } -  - void create() - { -  // Register localization projects - #define __REG_PROJ Locale.register_project -  __REG_PROJ("roxen_""start", "translations/%L/roxen_start.xml"); -  __REG_PROJ("roxen_""config", "translations/%L/roxen_config.xml"); -  __REG_PROJ("roxen_""message", "translations/%L/roxen_message.xml"); -  __REG_PROJ("admin_""tasks", "translations/%L/admin_tasks.xml"); -  Locale.set_default_project_path("translations/%L/%P.xml"); - #undef __REG_PROJ -  -  define_global_variables(); -  -  // for module encoding stuff -  -  // Pike 7.7 or later - we use the native LDAP module and link the -  // migration alias NewLDAP to it. -  add_constant ("NewLDAP", Protocols.LDAP); -  -  add_constant( "CFUserDBModule",config_userdb_module ); -  -  //add_constant( "ArgCache", ArgCache ); -  //add_constant( "roxen.load_image", load_image ); -  -  if (all_constants()["roxen"]) { -  error("Duplicate Roxen object!\n"); -  } -  -  // simplify dumped strings. -  add_constant( "roxen", this_object()); -  //add_constant( "roxen.decode_charset", decode_charset); -  - // add_constant( "DBManager", ((object)"base_server/dbs.pike") ); -  -  // This is currently needed to resolve the circular references in -  // RXML.pmod correctly. :P -  master()->resolv ("RXML.refs"); -  master()->resolv ("RXML.PXml"); -  master()->resolv ("RXML.PEnt"); -  foreach(({ "module.pmod","PEnt.pike", "PExpr.pike","PXml.pike", -  "refs.pmod","utils.pmod" }), string q ) -  dump( "etc/modules/RXML.pmod/"+ q ); -  dump( "etc/modules/RXML.pmod/module.pmod" ); -  master()->add_dump_constant ("RXML.empty_tag_set", -  master()->resolv ("RXML.empty_tag_set")); -  -  // Replace Val objects with versions extended for the rxml type system. -  Val->true = Roxen.true; -  Val->false = Roxen.false; -  Val->null = Roxen->sql_null = Roxen.null; -  -  // Already loaded. No delayed dump possible. -  dump( "etc/roxen_master.pike" ); -  dump( "etc/modules/Roxen.pmod" ); -  dump( "base_server/config_userdb.pike" ); -  dump( "base_server/disk_cache.pike" ); -  dump( "base_server/roxen.pike" ); -  dump( "base_server/basic_defvar.pike" ); -  dump( "base_server/newdecode.pike" ); -  dump( "base_server/read_config.pike" ); -  dump( "base_server/global_variables.pike" ); -  dump( "base_server/module_support.pike" ); -  dump( "base_server/socket.pike" ); -  dump( "base_server/cache.pike" ); -  dump( "base_server/supports.pike" ); -  dump( "base_server/hosts.pike"); -  dump( "base_server/language.pike"); -  -  DDUMP( "base_server/roxenlib.pike"); -  DDUMP( "etc/modules/Dims.pmod"); -  DDUMP( "config_interface/boxes/Box.pmod" ); -  dump( "base_server/html.pike"); -  -  add_constant( "RoxenModule", RoxenModule); -  add_constant( "ModuleInfo", ModuleInfo ); -  add_constant( "WebSocketAPI", WebSocketAPI); -  -  add_constant( "load", load); -  -  add_constant( "Roxen.set_locale", set_locale ); -  add_constant( "Roxen.get_locale", get_locale ); -  -  add_constant( "roxen.locale", locale ); -  //add_constant( "roxen.ImageCache", ImageCache ); -  - //int s = gethrtime(); -  _configuration = (program)"configuration"; -  dump( "base_server/configuration.pike" ); -  dump( "base_server/rxmlhelp.pike" ); -  -  // Override the one from prototypes.pike -  add_constant( "Configuration", _configuration ); - //report_debug( "[Configuration: %.2fms] ", (gethrtime()-s)/1000.0); - } -  - string get_locale( ) - { -  return locale->get(); - } -  - int set_u_and_gid (void|int from_handler_thread) - //! Set the uid and gid to the ones requested by the user. If the - //! sete* functions are available, and the define SET_EFFECTIVE is - //! enabled, the euid and egid is set. This might be a minor security - //! hole, but it will enable roxen to start CGI scripts with the - //! correct permissions (the ones the owner of that script have). - { - #ifndef __NT__ -  string u, g; -  int uid, gid; -  array pw; -  -  if (from_handler_thread && geteuid()) { -  // The euid switch in the backend thread worked here too, so -  // there's no need to do anything. - #ifdef TEST_EUID_CHANGE -  werror ("euid change effective in handler thread.\n"); - #endif -  return 1; -  } -  -  u=query("User"); -  sscanf(u, "%s:%s", u, g); - #if 0 -  // FIXME: Support caching the uid/gid in the setting -  // in case of lookup failure further below. -  sscanf(u, "%d", uid); -  sscanf(u, "%s(%d)", u, uid); -  if (g) { -  sscanf(g, "%d", gid); -  sscanf(g, "%s(%d)", g, gid); -  } - #endif /* 0 */ -  if(strlen(u)) -  { -  if(getuid()) -  { -  if (!from_handler_thread) -  report_error(LOC_M(24, "It is possible to change uid and gid only " -  "if the server is running as root.")+"\n"); -  } else { - #ifdef TEST_EUID_CHANGE -  if (Stdio.write_file ("rootonly", -  "Only root should be able to read this.\n", -  0600)) -  test_euid_change = 1; - #endif -  -  if (g) { - #if constant(getgrnam) -  pw = getgrnam (g); -  if (!pw) -  if (sscanf (g, "%d", gid)) pw = getgrgid (gid), g = (string) gid; -  else report_error ("Couldn't resolve group " + g + ".\n"), g = 0; -  if (pw) g = pw[0], gid = pw[2]; - #else -  if (!sscanf (g, "%d", gid)) -  report_warning ("Can't resolve " + g + " to gid on this system; " -  "numeric gid required.\n"); - #endif -  } -  -  pw = getpwnam (u); -  if (!pw) -  if (sscanf (u, "%d", uid)) pw = getpwuid (uid), u = (string) uid; -  else { -  report_error ("Couldn't resolve user " + u + ".\n"); -  return 0; -  } -  if (pw) { -  u = pw[0], uid = pw[2]; -  if (!g) gid = pw[3]; -  } -  - #ifdef THREADS -  Thread.MutexKey mutex_key; -  object threads_disabled; -  if (!from_handler_thread) { -  // If this is necessary from every handler thread, these -  // things are thread local and thus are no locks necessary. -  if (mixed err = catch { mutex_key = euid_egid_lock->lock(); }) -  master()->handle_error (err); -  threads_disabled = _disable_threads(); -  } - #endif -  - #if constant(seteuid) -  if (geteuid() != getuid()) seteuid (getuid()); - #endif -  - #if constant(initgroups) -  if (mixed err = catch { -  initgroups(pw[0], gid); -  // Doesn't always work - David. -  }) -  master()->handle_error (err); - #endif -  -  if (query("permanent_uid")) { - #if constant(setuid) -  if (g) { - # if constant(setgid) -  setgid(gid); -  if (getgid() != gid) { -  report_error(LOC_M(25, "Failed to set gid.")+"\n"); -  g = 0; -  } - # else -  if (!from_handler_thread) -  report_warning(LOC_M(26, "Setting gid not supported on this system.") -  +"\n"); -  g = 0; - # endif -  } -  setuid(uid); -  if (getuid() != uid) { -  report_error(LOC_M(27, "Failed to set uid.")+"\n"); -  u = 0; -  } -  if (u && !from_handler_thread) -  report_notice(CALL_M("setting_uid_gid_permanently", "eng") -  (uid, gid, u, g)); - #else -  if (!from_handler_thread) -  report_warning(LOC_M(28, "Setting uid not supported on this system.") -  +"\n"); -  u = g = 0; - #endif -  } -  else { - #if constant(seteuid) -  if (g) { - # if constant(setegid) -  setegid(gid); -  if (getegid() != gid) { -  report_error(LOC_M(29, "Failed to set effective gid.")+"\n"); -  g = 0; -  } - # else -  if (!from_handler_thread) -  report_warning(LOC_M(30, "Setting effective gid not supported on " -  "this system.")+"\n"); -  g = 0; - # endif -  } -  seteuid(uid); -  if (geteuid() != uid) { -  report_error(LOC_M(31, "Failed to set effective uid.")+"\n"); -  u = 0; -  } -  if (u && !from_handler_thread) -  report_notice(CALL_M("setting_uid_gid", "eng")(uid, gid, u, g)); - #else -  if (!from_handler_thread) -  report_warning(LOC_M(32, "Setting effective uid not supported on " -  "this system.")+"\n"); -  u = g = 0; - #endif -  } -  -  enable_coredumps(1); -  - #ifdef THREADS -  // Paranoia. -  mutex_key = 0; -  threads_disabled = 0; - #endif -  -  return !!u; -  } -  } - #endif -  return 0; - } -  - void reload_all_configurations() - { -  Configuration conf; -  array (object) new_confs = ({}); -  mapping config_cache = ([]); -  int modified; -  -  setvars(retrieve("Variables", 0)); -  -  foreach(list_all_configurations(), string config) -  { -  mixed err; -  Stat st; -  conf = find_configuration( config ); -  if(!(st = config_is_modified(config))) { -  if(conf) { -  config_cache[config] = config_stat_cache[config]; -  new_confs += ({ conf }); -  } -  continue; -  } -  modified = 1; -  config_cache[config] = st; -  if(conf) -  { -  conf->stop(); -  conf->invalidate_cache(); -  conf->create(conf->name); -  } else { -  if(err = catch -  { -  conf = enable_configuration(config); -  }) { -  string bt=describe_backtrace(err); -  report_error(LOC_M(33, "Error while enabling configuration %s%s"), -  config, (bt ? ":\n"+bt : "\n")); -  continue; -  } -  function sp = master()->resolv("DBManager.set_permission"); -  catch(sp( "docs", conf, 1 )); // the docs db can be non-existant -  sp( "local", conf, 2 ); -  } -  if(err = catch -  { -  conf->start( 0 ); -  conf->enable_all_modules(); -  }) { -  string bt=describe_backtrace(err); -  report_error(LOC_M(33, "Error while enabling configuration %s%s"), -  config, (bt ? ":\n"+bt : "\n" )); -  continue; -  } -  new_confs += ({ conf }); -  } -  -  foreach(configurations - new_confs, conf) -  { -  modified = 1; -  report_notice(LOC_M(34,"Disabling old configuration %s")+"\n", conf->name); -  conf->stop(); -  destruct(conf); -  } -  if(modified) { -  configurations = new_confs; -  fix_config_lookup(); -  config_stat_cache = config_cache; -  } - } -  - private mapping(string:Configuration) config_lookup = ([]); - // Maps config name to config object. -  - Thread.Local bootstrap_info = Thread.Local(); - // Used temporarily at configuration and module initialization to hold - // some info so that it's available even before create() in the - // configuration/module is called. -  - void fix_config_lookup() - { -  config_lookup = mkmapping (configurations->name, configurations); - #ifdef DEBUG -  if (sizeof (configurations) != sizeof (config_lookup)) -  error ("Duplicate configuration names in configurations array: %O", -  configurations->name); - #endif - } -  - Configuration get_configuration (string name) - //! Gets the configuration with the given identifier name. - { - #ifdef DEBUG -  if (sizeof (configurations) != sizeof (config_lookup)) -  error ("config_lookup out of synch with configurations.\n"); - #endif -  return config_lookup[name]; - } -  - private Configuration admin_config; -  - Configuration get_admin_configuration() - //! Returns the admin UI configuration, which is the one containing a - //! config_filesystem module instance. - { -  return admin_config; - } -  - Configuration enable_configuration(string name) - { - #ifdef DEBUG -  if (get_configuration (name)) -  error ("A configuration called %O already exists.\n", name); - #endif -  bootstrap_info->set (name); -  Configuration cf = _configuration(); -  configurations += ({ cf }); -  fix_config_lookup(); -  return cf; - } -  - void disable_configuration (string name) - { -  if (Configuration conf = config_lookup[ name ]) { -  configurations -= ({conf}); -  fix_config_lookup(); -  } - } -  - void remove_configuration (string name) - { -  disable_configuration (name); -  ::remove_configuration (name); - } -  - // Enable all configurations - void enable_configurations() - { -  array err; -  configurations = ({}); -  config_lookup = ([]); -  -  foreach(list_all_configurations(), string config) -  { -  int t = gethrtime(); -  Configuration conf_obj; -  int is_admin_config; -  report_debug("\nEnabling the configuration %s ...\n", config); -  if(err=catch { -  conf_obj = enable_configuration(config); -  is_admin_config = conf_obj->start(0); -  }) -  report_error("\n"+LOC_M(35, "Error while loading configuration %s%s"), -  config+":\n", describe_backtrace(err)+"\n"); -  if (is_admin_config) admin_config = conf_obj; -  report_debug("Enabled %s in %.1fms\n", config, (gethrtime()-t)/1000.0 ); -  } -  foreach( configurations, Configuration c ) -  { -  if(sizeof( c->registered_urls ) ) -  return; -  } -  report_fatal("No configurations could open any ports. Will shutdown.\n"); -  restart(0.0, 50); /* Actually a shutdown, but... */ - } -  - int all_modules_loaded; - void enable_configurations_modules() - { -  if( all_modules_loaded++ ) return; -  foreach(configurations, Configuration config) -  if(mixed err=catch( config->enable_all_modules() )) -  report_error(LOC_M(36, "Error while loading modules in " -  "configuration %s%s"), -  config->name+":\n", describe_backtrace(err)+"\n"); - } -  - mapping low_decode_image(string data) - { -  mapping w = Image._decode( data ); -  if( w->image ) return w; -  return 0; - } -  - constant decode_layers = Image.decode_layers; -  - mapping low_load_image(string f, RequestID id, void|mapping err) - { -  string data; -  if(id->misc->_load_image_called < 5) -  { -  // We were recursing very badly with the demo module here... -  id->misc->_load_image_called++; -  if(!(data=id->conf->try_get_file(f, id, 0, 0, 0, err))) -  { - #ifdef THREADS -  if (sscanf( f, "http://%[^/]", string host ) || -  sscanf (f, "https://%[^/]", host)) { -  mapping hd = ([ -  "User-Agent":version(), -  "Host":host, -  ]); -  if (mixed err = catch { -  data = Protocols.HTTP.get_url_data( f, 0, hd ); -  }) -  master()->handle_error (err); -  } - #endif -  if( !data ) -  return 0; -  } -  } -  id->misc->_load_image_called = 0; -  if(!data) return 0; -  return low_decode_image( data ); - } -  - array(Image.Layer)|mapping load_layers(string f, RequestID id, mapping|void opt) - { -  string data; -  mapping res = ([]); -  if(id->misc->_load_image_called < 5) -  { -  // We were recursing very badly with the demo module here... -  id->misc->_load_image_called++; -  if(!(data=id->conf->try_get_file(f, id, 0, 0, 0, res))) -  { - #ifdef THREADS -  if (sscanf( f, "http://%[^/]", string host ) || -  sscanf (f, "https://%[^/]", host)) { -  mapping hd = ([ -  "User-Agent":version(), -  "Host":host, -  ]); -  if (mixed err = catch { -  data = Protocols.HTTP.get_url_data( f, 0, hd ); -  }) -  master()->handle_error (err); -  } - #endif -  if( !data ) -  return res; -  } -  } -  id->misc->_load_image_called = 0; -  if(!data) return res; -  return decode_layers( data, opt ); - } -  - Image.Image load_image(string f, RequestID id, mapping|void err) - { -  mapping q = low_load_image( f, id, err ); -  if( q ) return q->img; -  return 0; - } -  - // do the chroot() call. This is not currently recommended, since - // roxen dynamically loads modules, all module files must be - // available at the new location. -  - private void fix_root(string to) - { - #ifndef __NT__ -  if(getuid()) -  { -  report_debug("It is impossible to chroot() if the server is not run as root.\n"); -  return; -  } -  -  if(!chroot(to)) -  { -  report_debug("Roxen: Cannot chroot to "+to+": "); - #if constant(real_perror) -  real_perror(); - #endif -  return; -  } -  report_debug("Root is now "+to+".\n"); - #endif - } -  - void create_pid_file(string where) - { - #ifndef __NT__ -  if(!where) return; - // where = replace(where, ({ "$pid", "$uid" }), - // ({ (string)getpid(), (string)getuid() })); -  -  object privs = Privs("Deleting old pid file."); -  r_rm(where); -  privs = 0; -  -  mixed err; -  -  // Note: The server lock file is often created by the start script, but -  // there is a race, so this code is here for paranoia reasons. -  if (!Stdio.exist(sprintf("/var/run/roxen-server.%d.pid", getpid())) && -  !Stdio.exist(sprintf("/tmp/roxen-server.%d.pid", getpid()))) { -  // NB: The following won't work if there's a wrapper process -  // for Roxen (eg started via gdb, truss or valgrind), -  // but that shouldn't matter much, since the pid lock file -  // won't be used in that case anyway. -  privs = Privs("Creating pid lock."); -  if (catch { -  // Try /var/run/ first. -  hardlink(sprintf("/var/run/roxen-start.%d.pid", getppid()), -  sprintf("/var/run/roxen-server.%d.pid", getpid())); -  } && (err = catch { -  // And then /tmp/. -  hardlink(sprintf("/tmp/roxen-start.%d.pid", getppid()), -  sprintf("/tmp/roxen-server.%d.pid", getpid())); -  })) { -  report_debug("Cannot create the pid lock file %O: %s", -  sprintf("/tmp/roxen-server.%d.pid", getpid()), -  describe_error(err)); -  } -  privs = 0; -  } -  if(err = catch { -  Stdio.write_file(where, sprintf("%d\n%d\n", getpid(), getppid())); -  }) -  report_debug("Cannot create the pid file %O: %s", -  where, describe_error (err)); - #endif - } -  - Pipe.pipe shuffle(Stdio.File from, Stdio.File to, -  Stdio.File|void to2, -  function(:void)|void callback) - { - #if constant(spider.shuffle) -  if(!to2) -  { -  object p = fastpipe( ); -  p->input(from); -  p->set_done_callback(callback); -  p->output(to); -  return p; -  } else { - #endif -  // 'fastpipe' does not support multiple outputs. -  Pipe.pipe p = Pipe.pipe(); -  if (callback) p->set_done_callback(callback); -  p->output(to); -  if(to2) p->output(to2); -  p->input(from); -  return p; - #if constant(spider.shuffle) -  } - #endif - } -  - // Dump a single thread. - void describe_thread (Thread.Thread thread) - { -  int hrnow = gethrtime(); -  string thread_descr = ""; -  if (string th_name = Roxen.thread_name(thread, 1)) -  thread_descr += " - " + th_name; -  if (int start_hrtime = thread_task_start_times[thread]) -  thread_descr += sprintf (" - busy for %.3fs", -  (hrnow - start_hrtime) / 1e6); -  report_debug(">> ### Thread 0x%x%s:\n", -  thread->id_number(), -  thread_descr); -  // Use roxenloader's original reference to describe_backtrace to sidestep -  // the background failure wrapper that's active in RUN_SELF_TEST. -  string th_bt = roxenloader.orig_predef_describe_bt(thread->backtrace()); -  -  // Expand any occurrences of: -  // Thread.Mutex(/*locked by 0x....*/) -  // to: -  // Thread.Mutex(/*locked by 0x.... - <thread name>*/) -  string bt_separator = "Thread.Mutex(/*locked by "; -  if (has_value(th_bt, bt_separator)) { -  array(string) bt_segs = th_bt / bt_separator; -  if (sizeof(bt_segs) > 1) { -  foreach (bt_segs; int idx; string bt_seg) { -  if (sscanf(bt_seg, "0x%[0-9a-fA-F]*/", string th_hex_addr)) { -  if (string th_name = Roxen.thread_name_from_addr("0x" + th_hex_addr)) { -  bt_segs[idx] = -  "0x" + th_hex_addr + " - " + th_name + -  bt_seg[sizeof(th_hex_addr) + 2..]; -  } -  } -  } -  th_bt = bt_segs * bt_separator; -  } -  } -  -  report_debug(">> " + replace (th_bt, "\n", "\n>> ") + "\n"); - } -  - // Dump all threads to the debug log. - void describe_all_threads (void|int ignored, // Might be the signal number. -  void|int(0..1) inhibit_threads_disabled) - { -  object threads_disabled; -  if (!inhibit_threads_disabled) -  // Disable all threads to avoid potential locking problems while we -  // have the backtraces. It also gives an atomic view of the state. -  threads_disabled = _disable_threads(); -  -  array(Thread.Thread) threads = all_threads(); -  -  report_debug("###### Describing all %d pike threads:\n>>\n", -  sizeof (threads)); -  -  threads = Array.sort_array ( -  threads, -  lambda (Thread.Thread a, Thread.Thread b) { -  // Backend thread first, otherwise in id order. -  if (a == backend_thread) -  return 0; -  else if (b == backend_thread) -  return 1; -  else -  return a->id_number() > b->id_number(); -  }); -  -  foreach (threads, Thread.Thread thread) { -  describe_thread (thread); -  } -  -  threads = 0; -  -  if (catch { -  array(array) queue = low_handle_queue->peek_array(); -  -  if (handle_queue != low_handle_queue) { -  queue += handle_queue->peek_array(); -  } -  -  // Ignore the handle thread shutdown marker, if any. -  queue -= ({0}); -  -  if (!sizeof (queue)) -  report_debug("###### No entries in the handler queue.\n"); -  else { -  report_debug("###### %d entries in the handler queue:\n>>\n", -  sizeof (queue)); -  foreach(queue; int i; array task) -  report_debug(">> %d: %s\n", i, -  replace (debug_format_queue_task (task), "\n", "\n>> ")); -  report_debug(">> \n"); -  } -  queue = 0; -  }) { -  report_debug("###### Handler queue busy.\n"); -  } -  -  if (catch { -  array queue = bg_queue->peek_array(); -  -  if (!sizeof (queue)) -  report_debug ("###### No entries in the background_run queue\n"); -  else { -  report_debug ("###### %d entries in the background_run queue:\n>>\n", -  sizeof (queue)); -  foreach (queue; int i; array task) -  report_debug (">> %d: %s\n", i, -  replace (debug_format_queue_task (task), "\n", "\n>> ")); -  report_debug (">> \n"); -  } -  queue = 0; -  }) { -  report_debug("###### background_run queue busy.\n"); -  } -  -  report_debug ("###### Thread and queue dumps done at %s\n", ctime (time())); -  -  threads_disabled = 0; -  - #ifdef DEBUG -  report_debug (RoxenDebug.report_leaks()); - #endif - } -  -  - // Dump threads by file polling. -  - constant cdt_poll_interval = 5; // Seconds. - constant cdt_dump_seq_interval = 60; -  - string cdt_directory, cdt_filename; -  - Thread.Thread cdt_thread; - int cdt_next_seq_dump; -  - void cdt_poll_file() - { -  Roxen.name_thread(this_thread(), "Dump Thread File Monitor"); -  while (this && query ("dump_threads_by_file")) { -  if (array(string) dir = r_get_dir (cdt_directory)) { -  if (has_value (dir, cdt_filename)) { -  r_rm (cdt_directory + "/" + cdt_filename); -  describe_all_threads(); -  } -  else if (time() >= cdt_next_seq_dump) { -  dir = glob (cdt_filename + ".*", dir); -  if (sizeof (dir)) { -  string file = dir[0]; -  r_rm (cdt_directory + "/" + file); -  describe_all_threads(); -  sscanf (file, cdt_filename + ".%d", int count); -  if (--count > 0) { -  open (cdt_directory + "/" + cdt_filename + "." + count, -  "cwt"); -  cdt_next_seq_dump = time (1) + cdt_dump_seq_interval; -  } -  } -  } -  } -  sleep (cdt_poll_interval); -  } -  Roxen.name_thread(this_thread(), 0); -  cdt_thread = 0; - } -  - void cdt_changed (Variable.Variable v) - { -  if (cdt_directory && v->query() && !cdt_thread) -  cdt_thread = Thread.thread_create (cdt_poll_file); - } -  - // ---------------------------------------- -  -  - constant dump = roxenloader.dump; -  - program slowpipe, fastpipe; -  - void initiate_argcache() - { -  int t = gethrtime(); -  report_debug( "Initiating argument cache ... \b"); -  if( mixed e = catch( argcache = ArgCache("arguments") ) ) -  { -  report_fatal( "Failed to initialize the global argument cache:\n" + - #ifdef DEBUG -  describe_backtrace(e) + - #else /* !DEBUG */ -  describe_error(e) + - #endif /* DEBUG */ -  "\n"); -  roxenloader.real_exit(1); -  } -  add_constant( "roxen.argcache", argcache ); -  report_debug("\bDone [%.2fms]\n", (gethrtime()-t)/1000.0); - } -  - #ifdef TIMERS - void show_timers() - { -  call_out( show_timers, 30 ); -  array a = values(timers); -  array b = indices( timers ); -  sort( a, b ); -  reverse(a); -  reverse(b); -  report_notice("Timers:\n"); -  for( int i = 0; i<sizeof(b); i++ ) -  report_notice( " %-30s : %10.1fms\n", b[i], a[i]/1000.0 ); -  report_notice("\n\n"); - } - #endif -  - void scan_certs(int|void force) - { -  foreach(query("CertGlobs"), string glob_pattern) { -  glob_pattern = String.trim_all_whites(glob_pattern); -  if (glob_pattern == "") continue; -  if (!has_value(glob_pattern, "*") && !has_value(glob_pattern, "?")) { -  CertDB.register_pem_file(glob_pattern); -  continue; -  } -  string dir = dirname(glob_pattern); -  string base = basename(glob_pattern); -  array(string) dirs = ({}); -  if (has_value(dir, "*") || has_value(dir, "?")) { -  // FIXME: Complicated case; expand the globbed dir. -  dirs = ({ dir }); -  } else { -  dirs = ({ dir }); -  } -  foreach(dirs, dir) { -  array(string) rdirs; -  if (has_prefix(dir, "/")) { -  // Absolute path. -  rdirs = ({ "/" }); -  } else { -  // lopen path -  rdirs = map(roxenloader.package_directories, roxen_path); -  } -  foreach(rdirs, string rdir) { -  array(string) paths = get_dir(combine_path(rdir, dir)); -  if (!paths) { - #ifdef SSL3_DEBUG -  if (errno() != System.ENOENT) { -  werror("Reading PEM dir %O failed: %s\n", -  combine_path(rdir, dir), strerror(errno())); -  } - #endif -  continue; -  } -  foreach(glob(base, paths), string fname) { - #ifdef SSL3_DEBUG -  werror("Found PEM file %O, matching %O.\n", -  Stdio.append_path(dir, fname), glob_pattern); - #endif -  CertDB.register_pem_file(Stdio.append_path(dir, fname)); -  } -  } -  } -  } -  CertDB.refresh_all_pem_files(force); - } -  - protected BackgroundProcess scan_certs_process; -  - // Start a background process that scan for new certs every 10 minutes. - protected void start_scan_certs() - { -  if (scan_certs_process) return; -  -  scan_certs_process = BackgroundProcess(600, scan_certs); - } -  - protected void stop_scan_certs() - { -  if (scan_certs_process) { -  scan_certs_process->stop(); -  scan_certs_process = UNDEFINED; -  } - } -  - protected class GCTimestamp - { -  array self_ref; -  protected void create() {self_ref = ({this_object()});} -  protected void destroy() { -  werror ("GC runs at %s", ctime(time())); -  GCTimestamp(); -  } - } -  - int log_gc_timestamps; - int log_gc_histogram; - int log_gc_verbose; - int log_gc_cycles; -  - string format_cycle (array(mixed) cycle) - { -  array(string) string_parts = ({}); -  foreach (cycle; int pos; mixed val) { -  string formatted; -  -  if (arrayp (val)) { -  formatted = sprintf ("array(%d)", sizeof (val)); -  } else if (mappingp (val)) { -  formatted = sprintf ("mapping(%d)", sizeof (val)); -  } else if (multisetp (val)) { -  formatted = sprintf ("multiset(%d)", sizeof (val)); -  } else { -  formatted = sprintf ("%O", val); -  } -  -  /* Identify object/mapping/array index of the next element in the cycle. */ -  mixed next_val; -  if (pos < sizeof (cycle) - 1) { -  next_val = cycle[pos + 1]; -  } else { -  next_val = cycle[0]; -  } -  -  if (multisetp(val)) { -  formatted += "[[index]]"; -  } else { -  // NB: This catch is to handle the case where val is an object -  // that implements lfun::_indices() and/or lfun::_values() -  // that throw errors. -  if (catch { -  array(mixed) inds = indices(val); -  array(mixed) vals = values(val); -  int i = search(vals, next_val); -  if (i >= 0) { -  // Found. -  if (intp(inds[i])) { -  formatted += sprintf("[%d]", inds[i]); -  } else if (stringp(inds[i])) { -  if (sizeof(inds[i]) < 100) { -  formatted += sprintf("[%O]", inds[i]); -  } else { -  formatted += sprintf("[string(len: %d)]", sizeof(inds[i])); -  } -  } else { -  formatted += sprintf("[%t]", inds[i]); -  } -  } else { -  i = search(inds, next_val); -  if (i >= 0) { -  formatted += "[[index]]"; -  } else if (objectp(val)) { -  formatted += "->protected"; -  } -  } -  }) { -  formatted += "[[broken]]"; -  } -  } -  -  string_parts += ({ formatted }); -  } -  -  return string_parts * " ==> "; - } -  - void reinstall_gc_callbacks() - { -  mapping(string:mixed) gc_params = ([ "pre_cb": 0, -  "post_cb": 0, -  "destruct_cb": 0, -  "done_cb": 0 ]); -  -  int gc_start; -  -  // mapping from program name (as reported by sprintf/%O) to number of -  // GC-destructed objects. Only valid in the GC's done_cb below. -  mapping(string:int) gc_histogram = ([]); -  -  // mapping from program name (as reported by sprintf/%O) to flag -  // indicating whether a cycle has been reported for this program in -  // the current GC report round. Cleared on every GC restart. -  mapping(string:int(0..1)) reported_cycles = ([]); -  -  if (log_gc_timestamps || log_gc_histogram || log_gc_verbose || -  log_gc_cycles) { -  gc_params->pre_cb = -  lambda() { -  gc_start = gethrtime(); -  gc_histogram = ([]); -  reported_cycles = ([]); -  werror("GC runs at %s", ctime(time())); -  }; -  -  gc_params->post_cb = -  lambda() { -  werror("GC done after %dms\n", -  (gethrtime() - gc_start) / 1000); -  }; -  -  if (log_gc_histogram || log_gc_verbose || log_gc_cycles) { -  gc_params->destruct_cb = -  lambda(object o) { -  // NB: These calls to sprintf(%O) can -  // take significant time. -  string id = -  sprintf("%O", object_program(o)); -  gc_histogram[id]++; -  if (log_gc_verbose) { -  werror("GC cyclic reference in %O.\n", -  o); -  } -  -  if (log_gc_cycles && !reported_cycles[id]) { -  reported_cycles[id] = 1; -  if (array(mixed) cycle = Pike.identify_cycle(o)) { -  werror ("GC cycle:\n%s\n", format_cycle (cycle)); -  } -  } -  }; -  } -  -  gc_params->done_cb = -  lambda(int n) { -  if (!n) return; -  werror("GC zapped %d things.\n", n); -  -  if (log_gc_histogram) { -  mapping h = gc_histogram; -  gc_histogram = ([]); -  if (!sizeof(h)) return; -  array i = indices(h); -  array v = values(h); -  sort(v, i); -  werror("GC histogram:\n"); -  foreach(reverse(i)[..9], string p) { -  werror("GC: %s: %d\n", p, h[p]); -  } -  } -  }; -  } -  -  Pike.gc_parameters(gc_params); - } -  - array argv; - int main(int argc, array tmp) - { -  // __builtin.gc_parameters((["enabled": 0])); -  argv = tmp; -  tmp = 0; -  - #if 0 -  Thread.thread_create (lambda () { -  while (1) { -  sleep (10); -  describe_all_threads(); -  } -  }); - #endif -  -  -  - #ifdef LOG_GC_TIMESTAMPS -  log_gc_timestamps = 1; - #endif - #ifdef LOG_GC_HISTOGRAM -  log_gc_histogram = 1; - #endif - #ifdef LOG_GC_VERBOSE -  log_gc_verbose = 1; - #endif - #ifdef LOG_GC_CYCLES -  log_gc_cycles = 1; - #endif -  -  reinstall_gc_callbacks(); -  -  // For RBF -  catch(mkdir(getenv("VARDIR") || "../var")); -  -  dbm_cached_get = master()->resolv( "DBManager.cached_get" ); -  -  dbm_cached_get( "local" )-> -  query( "CREATE TABLE IF NOT EXISTS " -  "compiled_formats (" -  " md5 CHAR(32) not null primary key," -  " full BLOB not null," -  " enc BLOB not null" -  ")" ); -  master()->resolv( "DBManager.is_module_table" ) -  ( 0, "local", "compiled_formats", -  "Compiled and cached log and security pattern code. "); -  master()->resolv( "DBManager.inhibit_backups" ) -  ( "local", "compiled_formats", ); -  -  slowpipe = ((program)"base_server/slowpipe"); -  fastpipe = ((program)"base_server/fastpipe"); -  dump( "etc/modules/DBManager.pmod" ); -  dump( "etc/modules/VFS.pmod" ); -  dump( "base_server/slowpipe.pike" ); -  dump( "base_server/fastpipe.pike" ); -  dump( "base_server/throttler.pike" ); -  -  if (!has_value (compat_levels, roxen_ver)) -  report_debug ("Warning: The current version %s does not exist in " -  "roxen.compat_levels.\n", roxen_ver); -  -  add_constant( "Protocol", Protocol ); - #ifdef TIMERS -  call_out( show_timers, 30 ); - #endif -  - #if constant(SSL.File) -  add_constant( "StartTLSProtocol", StartTLSProtocol ); -  add_constant( "SSLProtocol", SSLProtocol ); -  -  dbm_cached_get("roxen")-> -  query("CREATE TABLE IF NOT EXISTS cert_pem_files (" -  " id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, " -  // lopen()-compatible path to the PEM file. -  " path VARCHAR(2047) NOT NULL, " -  // Password to decode the PEM data (if any). -  " pass VARCHAR(255) NULL, " -  // mtime for the PEM file at last scan. -  // NULL if not valid. -  " mtime INT NULL, " -  // time at which the PEM file was last imported. -  // NULL if not imported yet. -  " itime INT NULL, " -  // Hash (currently SHA256) of PEM file data at last scan. -  // NULL if not imported. -  " hash VARBINARY(64) NULL, " -  // Index used when (un-)registering PEM files. -  " INDEX path (path), " -  // Index used when rescanning PEM files. -  " INDEX itime (itime)" -  ")"); -  master()->resolv( "DBManager.is_module_table" ) -  ( 0, "roxen", "certs", "Registry of known PEM files."); -  -  dbm_cached_get("roxen")-> -  query("CREATE TABLE IF NOT EXISTS certs (" -  " id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, " -  // Distinguished Name for the certified subject. -  " subject VARBINARY(255) NOT NULL, " -  // DN for the issuer of this certificate. -  " issuer VARBINARY(255) NOT NULL, " -  // Id for the cert that this cert is issued by. -  // NULL for self-signed or well known. -  " parent INT NULL, " -  // Id of the source PEM file. -  // NULL if stale. -  " pem_id INT NULL, " -  // Message number in the PEM file. -  // NULL if stale or refresh in progress. -  " msg_no INT DEFAULT 0 NULL, " -  // Expiry timestamp for the certificate. -  " expires INT NOT NULL, " -  // Data contained in the PEM. -  " data BLOB NOT NULL, " -  // Public key hash. -  " keyhash VARBINARY(64) NOT NULL, " -  // Index used when refreshing a PEM file. -  " INDEX (pem_id, msg_no), " -  // Index used when searching for certs matching a key. -  " INDEX keyhash (keyhash), " -  // Index used when searching for issuers and refreshing the cert. -  " INDEX subject (subject)," -  // Index used when searching for signed entities when -  // refreshing the cert. -  " INDEX issuer (issuer)" -  ")"); -  master()->resolv( "DBManager.is_module_table" ) -  ( 0, "roxen", "certs", "SSL/TLS Certificates."); -  -  dbm_cached_get("roxen")-> -  query("CREATE TABLE IF NOT EXISTS cert_keys (" -  " id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, " -  // Id of the source PEM file. -  // NULL if stale. -  " pem_id INT NULL, " -  // Message number in the PEM file. -  // NULL if stale or refresh in progress. -  " msg_no INT DEFAULT 0 NULL, " -  // Public key hash. -  // NULL if PEM decryption unsuccessful. -  " keyhash VARBINARY(64) NULL, " -  // Encrypted private key ASN.1. -  // Encrypted with AES.CCM keyed with SHA256(cert_secret + keyhash). -  // NULL if PEM decryption unsuccessful. -  " data BLOB NULL, " -  // Index used when refreshing a PEM file. -  " INDEX (pem_id, msg_no), " -  // Index used when searching for keys matching a cert. -  " INDEX keyhash (keyhash)" -  ")"); -  master()->resolv( "DBManager.is_module_table" ) -  ( 0, "roxen", "cert_keys", "SSL/TLS Private Keys."); -  -  dbm_cached_get("roxen")-> -  query("CREATE TABLE IF NOT EXISTS cert_keypairs (" -  " id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, " -  // Id for the cert. -  " cert_id INT NOT NULL, " -  // Id for the corresponding key. -  " key_id INT NOT NULL, " -  // Display name for the keypair. -  " name VARCHAR(255) NOT NULL DEFAULT '', " -  " INDEX (cert_id, key_id)" -  ")"); -  master()->resolv( "DBManager.is_module_table" ) -  ( 0, "roxen", "cert_keys", "SSL/TLS Key and Certificate matching."); - #endif -  -  dump( "etc/modules/Variable.pmod/module.pmod" ); -  dump( "etc/modules/Variable.pmod/Language.pike" ); -  dump( "etc/modules/Variable.pmod/Schedule.pike" ); -  -  foreach( glob("*.pike", get_dir( "etc/modules/Variable.pmod/")) -  -({"Language.pike", "Schedule.pike"}), string f ) -  DDUMP( "etc/modules/Variable.pmod/"+f ); -  -  DDUMP( "base_server/state.pike" ); -  DDUMP( "base_server/highlight_pike.pike" ); -  DDUMP( "base_server/wizard.pike" ); -  DDUMP( "base_server/proxyauth.pike" ); -  DDUMP( "base_server/module.pike" ); -  DDUMP( "base_server/throttler.pike" ); -  -  mark_fd(0, "Stdin"); -  mark_fd(1, "Stdout"); -  mark_fd(2, "Stderr"); -  -  once_mode = (int)Getopt.find_option(argv, "o", "once"); -  hot_reload_modules = Getopt.find_option(argv, 0, "module-hot-reload"); -  hot_reload_modules_conf = Getopt.find_option(argv, 0, "module-hot-reload-conf"); -  -  configuration_dir = -  Getopt.find_option(argv, "d",({"config-dir","configuration-directory" }), -  ({ "ROXEN_CONFIGDIR", "CONFIGURATIONS" }), "../configurations"); -  -  if(configuration_dir[-1] != '/') -  configuration_dir += "/"; -  -  restore_global_variables(); // restore settings... -  -  cache.set_total_size_limit (query ("mem_cache_size") * 1024 * 1024); -  -  if( query("replicate" ) ) -  { -  report_notice( "Enabling replication support\n"); -  add_constant( "WS_REPLICATE", 1 ); -  // Dumping of arg_cache_plugins generates problem when trying to -  // enable/disable argcache replication. -  // call_out( lambda() { -  // dump("arg_cache_plugins/replicate.pike"); -  // }, 2 ); -  } -  -  // Dangerous... -  mixed tmp_root; -  if(tmp_root = Getopt.find_option(argv, "r", "root")) fix_root(tmp_root); -  -  argv -= ({ 0 }); -  argc = sizeof(argv); -  -  fonts = ((program)"base_server/fonts.pike")(); -  -  DDUMP( "languages/abstract.pike" ); -  initiate_languages(query("locale")); -  -  cache_clear_deltas(); -  set_locale(); -  - #if constant(syslog) -  init_logger(); - #endif -  init_garber(); -  -  initiate_supports(); -  initiate_argcache(); -  init_configuserdb(); -  cache.init_session_cache(); -  -  // Report unhandled Promise rejections. -  Concurrent.on_failure(lambda(mixed err) -  { -  string description; -  if (objectp (err) && functionp(err->describe)) { -  description = err->describe(); -  } else if (arrayp (err) && sizeof (err) == 2) { -  description = describe_backtrace (err); -  } else { -  description = sprintf ("%O", err); -  } -  report_error("Unhandled error in Promise.\n" -  "Error: %s\n", description); -  }); -  -  protocols = build_protocols_mapping(); -  -  int t = gethrtime(); -  report_debug("Searching for pike-modules directories ... \b"); -  foreach( find_all_pike_module_directories( ), string d ) -  master()->add_module_path( d ); -  report_debug("\bDone [%dms]\n", (gethrtime()-t)/1000 ); -  - #ifdef SMTP_RELAY -  smtp_relay_start(); - #endif /* SMTP_RELAY */ -  - #ifdef SNMP_AGENT -  //SNMPagent start -  report_debug("SNMPagent configuration checking ... \b"); -  if(query("snmp_agent")) { -  // enabling SNMP agent -  snmpagent = SNMPagent(); -  snmpagent->enable(); -  report_debug("\benabled.\n"); -  snmpagent->start_trap(); -  -  } else -  report_debug("\bdisabled.\n"); - #endif // SNMP_AGENT -  -  backend_thread = this_thread(); - #ifdef THREADS -  Roxen.name_thread( backend_thread, "Backend" ); - #else -  report_debug("\n" -  "WARNING: Threads not enabled!\n" -  "\n"); - #endif /* THREADS */ -  -  foreach(({ "testca.pem", "demo_certificate.pem" }), string file_name) { -  if (!sizeof(roxenloader.package_directories)) break; -  CertDB.register_pem_file(file_name); -  string cert; -  if (lfile_path(file_name) == file_name) { -  file_name = roxen_path (roxenloader.package_directories[0] + "/" + -  file_name); -  report_notice("Generating a new certificate %s...\n", file_name); -  cert = Roxen.generate_self_signed_certificate("*"); - #if constant(Standards.X509) -  } else { -  file_name = roxen_path (lfile_path(file_name)); -  -  // Check if we need to upgrade the cert. -  // -  // Certificates generated by old versions of Pike were -  // plain X.509v1, while certificates generated by Pike 8.0 -  // and later are X.509v3 with some required extensions. -  -  string old_cert = Stdio.read_bytes(file_name); -  if (!old_cert) { -  report_error("Failed to read certificate %s.\n", file_name); -  continue; -  } -  -  // Note: set_u_and_gid() hasn't been called yet, -  // so there's no need for Privs. -  Standards.PEM.Messages msgs = Standards.PEM.Messages(old_cert); -  -  int upgrade_needed; -  -  foreach(msgs->parts; string part; array(Standards.PEM.Message) msg) { -  if (!has_suffix(part, "CERTIFICATE")) continue; -  Standards.X509.TBSCertificate tbs = -  Standards.X509.decode_certificate(msg[0]->body); -  upgrade_needed = (tbs->version < 3); -  if (!upgrade_needed) { -  // NB: This stuff is not only to work around that -  // Standards.X509.algorithms is protected, but -  // also to avoid having to know about how the -  // X509 algorithm sequence is structured. -  class HashAlgVerifier { -  inherit Standards.X509.Verifier; -  constant type = "hash"; -  class HashPKC { -  inherit Crypto.Sign.State; -  int(0..1) pkcs_verify(string(8bit) msg, -  Crypto.Hash h, -  string(8bit) sign) -  { -  // Disallow SHA1 and shorter. -  // Allow SHA224 and longer. -  return h && (h->digest_size() >= 28); -  } -  }; -  protected void create() -  { -  pkc = HashPKC(); -  } -  }; -  upgrade_needed = !HashAlgVerifier()->verify(tbs->algorithm, "", ""); -  } -  break; -  } -  -  if (!upgrade_needed || (sizeof(msgs->parts) != 2)) continue; -  -  // NB: We reuse the old key. -  Crypto.Sign key; -  foreach(msgs->parts; string part; array(Standards.PEM.Message) msg) { -  if (!has_suffix(part, "PRIVATE KEY")) continue; -  if (msg[0]->headers["dek-info"]) { -  // Not supported here. -  break; -  } -  key = Standards.X509.parse_private_key(msg[0]->body); -  } -  if (!key) continue; -  -  report_notice("Renewing certificate: %O...\n", file_name); -  cert = Roxen.generate_self_signed_certificate("*", key); - #endif /* constant(Standards.X509) */ -  } -  -  if (cert) { -  // Note: set_u_and_gid() hasn't been called yet, -  // so there's no need for Privs. -  Stdio.File file = Stdio.File(); -  if (!file->open(file_name, "wtc", 0600)) { -  report_error("Couldn't create certificate file %s: %s\n", file_name, -  strerror (file->errno())); -  } else if (file->write(cert) != sizeof(cert)) { -  rm(file_name); -  report_error("Couldn't write certificate file %s: %s\n", file_name, -  strerror (file->errno())); -  } -  } -  } -  - #ifdef THREADS -  start_low_handler_threads(); - #endif /* THREADS */ -  -  // Update the certificate registry before opening any ports. -  // NB: Force all certificate files to be reread and reparsed. -  scan_certs(1); -  -  enable_configurations(); -  -  string pid_file = Getopt.find_option(argv, "p", "pid-file"); -  if (pid_file && query("permanent_uid")) rm(pid_file); -  -  set_u_and_gid(); // Running with the right [e]uid:[e]gid from this point on. -  -  create_pid_file(pid_file); -  -  // Done before the modules are dumped. -  - #ifdef RUN_SELF_TEST -  enable_configurations_modules(); - #else -  if( Getopt.find_option( argv, 0, "no-delayed-load" )) -  enable_configurations_modules(); -  else -  foreach( configurations, Configuration c ) -  if( c->query( "no_delayed_load" ) ) -  c->enable_all_modules(); - #endif // RUN_SELF_TEST -  - #ifdef THREADS -  start_handler_threads(); - #if constant(Filesystem.Monitor.basic) -  start_fsgarb(); - #endif - #endif /* THREADS */ -  -  start_scan_certs(); -  start_hourly_maintenance(); -  - #ifdef TEST_EUID_CHANGE -  if (test_euid_change) { -  Stdio.File f = Stdio.File(); -  if (f->open ("rootonly", "r") && f->read()) -  werror ("Backend thread can read rootonly\n"); -  else -  werror ("Backend thread can't read rootonly\n"); -  } - #endif -  -  // Signals which need to be ignored -  foreach( ({ "SIGPIPE" }), string sig) -  catch( signal(signum(sig), 0) ); -  -  // Signals which cause a restart (exitcode != 0) -  foreach( ({ "SIGINT", "SIGTERM" }), string sig) -  catch( signal(signum(sig), async_sig_start(exit_when_done,0)) ); -  -  catch(signal(signum("SIGHUP"),async_sig_start(reload_all_configurations,1))); -  -  // Signals which cause Roxen to dump the thread state -  foreach( ({ "SIGBREAK", "SIGQUIT", - #ifdef ENABLE_SIGWINCH -  "SIGWINCH", - #endif -  }), string sig) -  catch( signal(signum(sig),async_sig_start(describe_all_threads,-1))); -  -  start_time=time(); // Used by the "uptime" info later on. -  -  restart_suicide_checker(); -  -  { -  array(string) splitdir = roxen_path ("$LOGFILE") / "/"; -  cdt_filename = splitdir[-1]; -  cdt_directory = splitdir[..sizeof (splitdir) - 2] * "/"; -  if (has_suffix (cdt_filename, ".1")) -  cdt_filename = cdt_filename[..sizeof (cdt_filename) - 3]; -  cdt_filename += ".dump_threads"; -  cdt_changed (getvar ("dump_threads_by_file")); -  } -  - #ifndef NO_SLOW_REQ_BT -  slow_req_count_changed(); -  slow_req_timeout_changed(); -  slow_be_timeout_changed(); - #endif -  - #ifdef ROXEN_DEBUG_MEMORY_TRACE -  restart_roxen_debug_memory_trace(); - #endif -  - #ifndef __NT__ -  restart_if_stuck( 0 ); - #endif - #ifdef __RUN_TRACE -  trace(1); - #endif -  return -1; - } -  - void check_commit_suicide() - { - #ifdef SUICIDE_DEBUG -  werror("check_commit_suicide(): Engage:%d, schedule: %d, time: %d\n" -  " Schedule: %s", -  query("suicide_engage"), -  getvar("suicide_schedule")->get_next( query("last_suicide")), -  time(), -  ctime(getvar("suicide_schedule")->get_next( query("last_suicide")))); - #endif /* SUICIDE_DEBUG */ -  if (query("suicide_engage")) { -  int next = getvar("suicide_schedule") -  ->get_next( query("last_suicide") ); -  if (next >= 0) -  if (next <= time(1)) { -  report_notice("Auto Restart triggered.\n"); -  set( "last_suicide", time(1) ); -  save( ); -  restart(); -  } else { -  call_out(check_commit_suicide, next - time(1)); -  } -  } - } -  - void check_suicide( ) - { - #ifdef SUICIDE_DEBUG -  werror("check_suicide(): Engage:%d, schedule: %d, time: %d\n" -  " Schedule: %s", -  query("suicide_engage"), -  getvar("suicide_schedule")->get_next( query("last_suicide")), -  time(), -  ctime(getvar("suicide_schedule")->get_next( query("last_suicide")))); - #endif /* SUICIDE_DEBUG */ -  if (query("suicide_engage")) { -  int next = getvar("suicide_schedule") -  ->get_next( query("last_suicide") ); -  if( !query("last_suicide") || (next >= 0 && next <= time()) ) -  { - #ifdef SUICIDE_DEBUG -  werror("Next suicide is in the past or last time not set. Reseting.\n"); - #endif -  set( "last_suicide", time() ); -  save( ); -  } -  } - } -  - void restart_suicide_checker() - { -  remove_call_out(check_commit_suicide); -  remove_call_out(check_suicide); -  call_out(check_suicide, 60); -  call_out(check_commit_suicide, 180); // Minimum uptime: 3 minutes. - } -  - #ifdef ROXEN_DEBUG_MEMORY_TRACE - protected object roxen_debug_info_obj; - void restart_roxen_debug_memory_trace() - { -  remove_call_out(restart_roxen_debug_memory_trace); -  -  if (!roxen_debug_info_obj) { -  roxen_debug_info_obj = ((program)"config_interface/actions/debug_info.pike" - )(); -  } -  int t = time(); -  string html = roxen_debug_info_obj->parse((["real_variables":([])])); -  if (!Stdio.is_dir("../var/debug")) { -  mkdir("../var/debug"); -  } -  Stdio.write_file(sprintf("../var/debug/memory_info_%d.rxml", t), html); -  call_out(restart_roxen_debug_memory_trace, 5); - } - #endif -  - // Called from the administration interface. - string check_variable(string name, mixed value) - { -  switch(name) -  { - #ifndef __NT__ -  case "abs_engage": -  if (value) -  // Make sure restart_if_stuck is called from the backend thread. -  call_out(restart_if_stuck, 0, 1); -  else { -  remove_call_out(restart_if_stuck); -  alarm(0); -  } -  break; -  case "abs_timeout": -  if (value < 0) { -  return "The timeout must be >= 0 minutes."; -  } -  break; - #endif -  -  case "suicide_schedule": -  case "suicide_engage": -  restart_suicide_checker(); -  break; -  - #ifdef SNMP_AGENT -  case "snmp_agent": -  if (value && !snmpagent) { -  report_notice("SNMPagent enabling ...\n"); -  snmpagent = SNMPagent(); -  snmpagent->enable(); -  snmpagent->start_trap(); -  } -  if (!value && objectp(snmpagent)) { -  report_notice("SNMPagent disabling ...\n"); -  snmpagent->stop_trap(); -  snmpagent->disable(); -  snmpagent = 0; -  } -  break; - #endif // SNMP_AGENT -  -  } - } -  - int is_ip(string s) - { -  return s && -  (sscanf(s,"%*d.%*d.%*d.%*d%*c")==4 || // IPv4 -  has_value (s, ":")); // IPv6 - } -  - //! @ignore - DECLARE_OBJ_COUNT; - //! @endignore -  - protected string _sprintf( ) - { -  return "roxen()" + OBJ_COUNT; - } -  -  - // Logging -  - class LogFormat // Note: Dumping won't work if protected. - { -  protected string url_encode (string str) -  { -  // Somewhat like Roxen.http_encode_url, but only encode enough -  // chars to avoid ambiguity in typical log formats. Notably, UTF-8 -  // encoded chars aren't URL encoded too, to make the log easier to -  // view in any UTF-8 aware editor or viewer. -  return replace ( -  string_to_utf8 (str), ({ -  // Control chars. -  "\000", "\001", "\002", "\003", "\004", "\005", "\006", "\007", -  "\010", "\011", "\012", "\013", "\014", "\015", "\016", "\017", -  "\020", "\021", "\022", "\023", "\024", "\025", "\026", "\027", -  "\030", "\031", "\032", "\033", "\034", "\035", "\036", "\037", -  "\177", -  // The escape char. -  "%", -  // Basic log format separators. -  " ", "\"", "'", -  }), -  ({ -  "%00", "%01", "%02", "%03", "%04", "%05", "%06", "%07", -  "%08", "%09", "%0a", "%0b", "%0c", "%0d", "%0e", "%0f", -  "%10", "%11", "%12", "%13", "%14", "%15", "%16", "%17", -  "%18", "%19", "%1a", "%1b", "%1c", "%1d", "%1e", "%1f", -  "%7f", -  "%25", -  "%20", "%22", "%27", -  })); -  } -  -  protected int rusage_time; -  protected mapping(string:int) rusage_data; -  protected void update_rusage() -  { -  if(!rusage_data || time(1) != rusage_time) -  { -  rusage_data = (["utime": 1, "stime": 1]) & System.getrusage(); -  rusage_time = time(1); -  } -  } -  -  protected int server_cputime() -  { -  update_rusage(); -  if(rusage_data) -  return rusage_data->utime + rusage_data->stime; -  return 0; -  } -  -  protected int server_usertime() -  { -  update_rusage(); -  if(rusage_data) -  return rusage_data->utime; -  return 0; -  } -  -  protected int server_systime() -  { -  update_rusage(); -  if(rusage_data) -  return rusage_data->stime; -  return 0; -  } -  -  protected string std_date(mapping(string:int) ct) { -  return(sprintf("%04d-%02d-%02d", -  1900+ct->year,ct->mon+1, ct->mday)); -  } -  -  protected string std_time(mapping(string:int) ct) { -  return(sprintf("%02d:%02d:%02d", -  ct->hour, ct->min, ct->sec)); -  } -  -  // CERN date formatter. Note similar code in Roxen.pmod. -  -  protected constant months = ({ "Jan", "Feb", "Mar", "Apr", "May", "Jun", -  "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }); -  -  protected int chd_lt; -  protected string chd_lf; -  -  protected string cern_http_date(int t, mapping(string:int) ct) -  { -  if( t == chd_lt ) -  // Interpreter lock assumed here. -  return chd_lf; -  -  string c; -  int tzh = ct->timezone/3600; -  if(tzh > 0) -  c="-"; -  else { -  tzh = -tzh; -  c="+"; -  } -  -  c = sprintf("%02d/%s/%04d:%02d:%02d:%02d %s%02d00", -  ct->mday, months[ct->mon], 1900+ct->year, -  ct->hour, ct->min, ct->sec, c, tzh); -  -  chd_lt = t; -  // Interpreter lock assumed here. -  chd_lf = c; -  -  return c; -  } -  -  protected string host_ip_to_int(string s) -  { -  int a, b, c, d; -  sscanf(s, "%d.%d.%d.%d", a, b, c, d); -  return sprintf("%c%c%c%c",a, b, c, d); -  } -  -  protected string extract_user(string from) -  { -  array tmp; -  if (!from || sizeof(tmp = from/":")<2) -  return "-"; -  return tmp[0]; // username only, no password -  } -  -  protected string get_forwarded_field(RequestID id, string field) -  { -  foreach(id->misc->forwarded || ({}), array(string|int) segment) { -  if (!arrayp(segment) || sizeof(segment) < 3) continue; -  if (segment[0] != field || segment[1] != '=') continue; -  return MIME.quote(segment[2..]); -  } -  return "-"; -  } -  -  void log_access( function do_write, RequestID id, mapping file ); -  -  void log_event (function do_write, string facility, string action, -  string resource, mapping(string:mixed) info); -  -  protected void do_async_write( string host, string data, -  string ip, function c ) -  { -  if( c ) -  c( replace( data, "\4711", (host||ip) ) ); -  } - } -  - protected mapping(string:function) compiled_log_access = ([ ]); - protected mapping(string:function) compiled_log_event = ([ ]); -  - #define LOG_ASYNC_HOST 1 - #define LOG_NEED_COOKIES 2 - #define LOG_NEED_TIMESTAMP 4 - #define LOG_NEED_LTIME (8 | LOG_NEED_TIMESTAMP) - #define LOG_NEED_GTIME (16 | LOG_NEED_TIMESTAMP) -  - // Elements of a format array arr: - // arr[0]: sprintf format for access logging (run_log_format). - // arr[1]: Code for the corresponding sprintf argument of arr[0]. - // arr[2]: sprintf format for event logging (run_log_event_format). - // May be 0 to reuse arr[0] and arr[1]. - // May be 1 to indicate that an attempt is made to look up the - // variable in the info mapping. If it isn't found then arr[3] is - // used as fallback. The sprintf format string is always "%s" in - // this case. - // arr[3]: Code for the corresponding sprintf argument of arr[2]. - // arr[4]: Flags. -  - protected constant formats = ([ -  -  // Used for both access and event logging -  "date": ({"%s", "std_date (ltime)", 0, 0, LOG_NEED_LTIME}), -  "time": ({"%s", "std_time (ltime)", 0, 0, LOG_NEED_LTIME}), -  "cern-date": ({"%s", "cern_http_date (timestamp, ltime)", -  0, 0, LOG_NEED_LTIME}), -  "utc-date": ({"%s", "std_date (gtime)", 0, 0, LOG_NEED_GTIME}), -  "utc-time": ({"%s", "std_time (gtime)", 0, 0, LOG_NEED_GTIME}), -  "bin-date": ({"%4c", "timestamp", 0, 0, LOG_NEED_TIMESTAMP}), -  // FIXME: There is no difference between $resource and $full-resource. -  "resource": ({"%s", ("(string)" -  "(request_id->raw_url||" -  " (request_id->misc->common&&" -  " request_id->misc->common->orig_url)||" -  " request_id->not_query)"), -  "%s" , "url_encode (resource)", 0}), -  "server-uptime": ({"%d", "max(1, timestamp - roxen->start_time)", -  0, 0, 0}), -  "server-cputime": ({"%d", "server_cputime()", 0, 0, 0}), -  "server-usertime": ({"%d", "server_usertime()", 0, 0, 0}), -  "server-systime": ({"%d", "server_systime()", 0, 0, 0}), -  -  // Used for access logging -  "host": ({"\4711" /* unlikely to occur normally */, 0, -  1, "\"-\"", LOG_ASYNC_HOST}), -  "vhost": ({"%s", "(request_id->misc->host||\"-\")", -  1, "\"-\"", 0}), -  "ip-number": ({"%s", "(string)request_id->remoteaddr", -  1, "\"0.0.0.0\"", 0}), -  "bin-ip-number": ({"%s", "host_ip_to_int(request_id->remoteaddr)", -  1, "\"\\0\\0\\0\\0\"", 0}), -  "link-layer": ({"%s", "request_id->query_link_layer()", -  1, "\"-\"", 0}), -  "cipher-suite": ({"%s", "request_id->query_cipher_suite()", -  1, "\"-\"", 0}), -  "method": ({"%s", "(string)request_id->method", -  1, "\"-\"", 0}), -  "full-resource": ({"%s", ("(string)" -  "(request_id->raw_url||" -  " (request_id->misc->common&&" -  " request_id->misc->common->orig_url)||" -  " request_id->not_query)"), -  "%s" , "url_encode (resource)", 0}), -  "cs-uri-stem": ({"%s", ("(string)" -  "((request_id->misc->common&&" -  " request_id->misc->common->orig_url)||" -  " request_id->not_query||" -  " (request_id->raw_url && " -  " (request_id->raw_url/\"?\")[0])||" -  " \"-\")"), -  "%s" , "url_encode (resource)", 0}), -  "cs-uri-query": ({"%s", "(string)(request_id->query||\"-\")", -  1, "\"-\"", 0}), -  // FIXME: There is no difference between $real-resource and -  // $real-full-resource. -  "real-resource": ({"%s", ("(string)(request_id->raw_url||" -  " request_id->not_query)"), -  "%s" , "url_encode (resource)", 0}), -  "real-full-resource": ({"%s", ("(string)(request_id->raw_url||" -  " request_id->not_query)"), -  "%s" , "url_encode (resource)", 0}), -  "real-cs-uri-stem": ({"%s", ("(string)(request_id->not_query||" -  " (request_id->raw_url && " -  " (request_id->raw_url/\"?\")[0])||" -  " \"-\")"), -  "%s" , "url_encode (resource)", 0}), -  "protocol": ({"%s", "(string)request_id->prot", 1, "\"-\"", 0}), -  "scheme": ({"%s", "(string)((request_id->port_obj && " -  "request_id->port_obj->prot_name) || \"-\")", -  1, "\"-\"", 0 }), -  "response": ({"%d", "(int)(file->error || 200)", 1, "\"-\"", 0}), -  "bin-response": ({"%2c", "(int)(file->error || 200)", 1, "\"\\0\\0\"", 0}), -  "length": ({"%d", "(int)file->len", 1, "\"0\"", 0}), -  "bin-length": ({"%4c", "(int)file->len", 1, "\"\\0\\0\\0\\0\"", 0}), -  "request-length": ({"%d", ("(int)(request_id->raw_bytes - " -  "(sizeof(request_id->leftovers || \"\") + " -  "request_id->misc->len))"), -  1, "\"-\"", 0 }), -  "bin-request-length": ({"%4c", ("(int)(request_id->raw_bytes - " -  "(sizeof(request_id->leftovers || \"\") + " -  "request_id->misc->len))"), -  1, "\"\\0\\0\\0\\0\"", 0 }), -  "request-data-length": ({"%d", "(int)request_id->misc->len", -  1, "\"-\"", 0}), -  "bin-request-data-length": ({"%4c", "(int)request_id->misc->len", -  1, "\"\\0\\0\\0\\0\"", 0}), -  "queue-length": ({"%d", "(int) request_id->queue_length", -  1, "\"-\"", 0}), -  "request-time": ({"%1.4f", ("(float)(gethrtime() - " -  " request_id->hrtime) /" -  "1000000.0"), -  1, "\"-\"", 0}), -  "protocol-time": ({"%1.4f", -  "(float) request_id->protocol_time / 1000000.0", -  1, "\"-\"", 0}), -  "queue-time": ({"%1.4f", -  "(float) request_id->queue_time / 1000000.0", -  1, "\"-\"", 0}), -  "handle-time": ({"%1.4f", -  "(float) request_id->handle_time / 1000000.0", -  1, "\"-\"", 0}), -  "handle-cputime": ({ - #if constant(System.CPU_TIME_IS_THREAD_LOCAL) -  "%1.4f", "(float) request_id->handle_vtime / 1000000.0", - #else -  "%s", "\"-\"", - #endif -  1, "\"-\"", 0}), -  "etag": ({"%s", "request_id->misc->etag || \"-\"", -  1, "\"-\"", 0}), -  "referrer": ({"%s", ("sizeof(request_id->referer||({}))?" -  "request_id->referer[0]:\"-\""), -  1, "\"-\"", 0}), -  "referer": ({"%s", ("sizeof(request_id->referer||({}))?" -  "request_id->referer[0]:\"-\""), -  1, "\"-\"", 0}), -  "user-agent": ({"%s", ("request_id->client?" -  "request_id->client*\"%20\":\"-\""), -  1, "\"-\"", 0}), -  "user-agent-raw": ({"%s", ("request_id->client?" -  "request_id->client*\" \":\"-\""), -  1, "\"-\"", 0}), -  "user": ({"%s", "extract_user( request_id->realauth )", -  1, "\"-\"", 0}), -  "user-id": ({"%s", ("(request_id->cookies&&" -  " request_id->cookies->RoxenUserID)||" -  "(request_id->misc->moreheads&&" -  " request_id->misc->" -  " moreheads[\"Set-Cookie\"]&&" -  " request_id->parse_cookies&&" -  " request_id->parse_cookies(" -  " request_id->misc->" -  " moreheads[\"Set-Cookie\"])->" -  " RoxenUserID)||" -  "\"0\""), -  1, "\"-\"", 0}), -  "content-type": ({"%s", ("(((arrayp(file->type) ? " -  "file->type[0] : file->type) " -  "|| \"-\") / \";\")[0]"), -  1, "\"-\"", 0}), -  "cookies": ({"%s", ("arrayp(request_id->request_headers->cookie)?" -  "request_id->request_headers->cookie*\";\":" -  "request_id->request_headers->cookie||\"\""), -  1, "\"-\"", 0}), -  "set-cookies": ({"%s", ("Array.uniq(" -  "request_id->get_response_headers(\"Set-Cookie\")+" -  "request_id->get_response_headers(\"set-cookie\"))" -  "*\";\""), -  1, "\"-\"", 0}), -  "cache-status": ({"%s", ("sizeof(request_id->cache_status||({}))?" -  "indices(request_id->cache_status)*\",\":" -  "\"nocache\""), -  1, "\"-\"", 0}), -  "eval-status": ({"%s", ("sizeof(request_id->eval_status||({}))?" -  "indices(request_id->eval_status)*\",\":" -  "\"-\""), -  1, "\"-\"", 0}), -  "protcache-cost": ({"%d", "request_id->misc->protcache_cost", -  1, "\"-\"", 0}), -  "forwarded": ({"%s", ("request_id->misc->forwarded ? " -  "MIME.quote(request_id->misc->forwarded *" -  " ({ ',' })) : \"-\""), -  1, "\"-\"", 0 }), -  "xff": ({"%s", "get_forwarded_field(request_id, \"for\")", -  1, "\"-\"", 0 }), -  -  // Used for event logging -  "facility": ({"-", 0, "%s", "facility", 0}), -  "action": ({"-", 0, "%s", "action", 0}), - ]); -  - void run_log_format( string fmt, function c, RequestID id, mapping file ) - { -  (compiled_log_access[ fmt ] || -  compile_log_format( fmt )->log_access) (c,id,file); - } -  - void run_log_event_format (string fmt, function cb, -  string facility, string action, string resource, -  mapping(string:mixed) info) - { -  (compiled_log_event[ fmt ] || -  compile_log_format( fmt )->log_event) (cb, facility, action, -  resource, info); - } -  - protected LogFormat compile_log_format( string fmt ) - { -  add_constant( "___LogFormat", LogFormat ); -  -  // Note similar code in compile_security_pattern. -  -  string kmd5 = md5( fmt ); -  -  object con = dbm_cached_get("local"); -  -  { -  array tmp = -  con->query("SELECT full,enc FROM compiled_formats WHERE md5=%s", kmd5 ); -  -  if( sizeof(tmp) && (tmp[0]->full == fmt) ) -  { -  LogFormat lf; -  if (mixed err = catch { -  lf = decode_value( tmp[0]->enc, master()->Decoder() )(); -  }) { -  if (describe_error (err) != -  "Cannot decode programs encoded with other pike version.\n") -  report_warning ("Decoding of dumped log format failed " -  "(will recompile): %s", describe_backtrace(err)); -  } -  -  if (lf && lf->log_access) { -  // Check that it's a new style log program (old ones have log() -  // instead of log_access()). -  compiled_log_access[fmt] = lf->log_access; -  compiled_log_event[fmt] = lf->log_event; -  return lf; -  } -  } -  } -  -  array parts = fmt/"$"; -  string a_format = parts[0], e_format = parts[0]; -  array a_args = ({}), e_args = ({}); -  int log_flags = 0; -  int add_nl = 1; -  - #define DO_ES(X) replace(X, ({"\\n", "\\r", "\\t", "%"}), \ -  ({ "\n", "\r", "\t", "%%" }) ) -  -  foreach( parts[1..], string part ) -  { -  sscanf (part, "%[-_a-zA-Z0-9]%s", string kwd, part); -  kwd = replace (kwd, "_", "-"); -  -  if (array(string|int) spec = formats[kwd]) { -  [string a_fmt, string a_code, -  string|int e_fmt, string e_code, int flags] = spec; -  string escaped = DO_ES (part); -  a_format += a_fmt + escaped; -  if( a_code ) a_args += ({ a_code }); -  if (!e_fmt) e_fmt = a_fmt, e_code = a_code; -  else if (e_fmt == 1) { -  e_fmt = "%s"; -  e_code = sprintf ("info && !zero_type (info[%O]) ? " -  "url_encode ((string) info[%O]) : (%s)", -  kwd, kwd, e_code); -  } -  e_format += e_fmt + escaped; -  if (e_code) e_args += ({e_code}); -  log_flags |= flags; -  continue; -  } -  -  switch (kwd) { -  case "char": -  if( sscanf( part, "(%d)%s", int c, part ) == 2 ) { -  string s = sprintf( "%"+(c<0?"-":"")+"c", abs( c ) )+DO_ES(part); -  a_format += s, e_format += s; -  continue; -  } -  break; -  case "wchar": -  if( sscanf( part, "(%d)%s", int c, part ) == 2 ) { -  string s = sprintf( "%"+(c<0?"-":"")+"2c", abs( c ) )+DO_ES(part); -  a_format += s, e_format += s; -  continue; -  } -  break; -  case "int": -  if( sscanf( part, "(%d)%s", int c, part ) == 2 ) { -  string s = sprintf( "%"+(c<0?"-":"")+"4c", abs( c ) )+DO_ES(part); -  a_format += s, e_format += s; -  continue; -  } -  break; -  case "": -  if( part[0] == '^' ) -  { -  string escaped = DO_ES(part[1..]); -  a_format += escaped, e_format += escaped; -  add_nl = 0; -  continue; -  } -  break; -  } -  -  a_format += "-" + DO_ES (part); -  -  // Any unknown variable is indexed from the info mapping for events. -  e_format += "%s" + DO_ES (part); -  e_args += ({sprintf ("info && !zero_type (info[%O]) ? " -  "url_encode ((string) info[%O]) : \"-\"", -  kwd, kwd)}); -  } -  if( add_nl ) a_format += "\n", e_format += "\n"; -  -  string a_func = #" -  void log_access( function callback, RequestID request_id, mapping file ) -  { -  if(!callback) return;"; -  string e_func = #" -  void log_event (function callback, string facility, string action, -  string resource, mapping(string:mixed) info) -  { -  if(!callback) return;"; -  -  if (log_flags & LOG_NEED_TIMESTAMP) { -  string c = #" -  int timestamp = time (1);"; -  a_func += c, e_func += c; -  } -  if (log_flags & LOG_NEED_LTIME) { -  string c = #" -  mapping(string:int) ltime = localtime (timestamp);"; -  a_func += c, e_func += c; -  } -  if (log_flags & LOG_NEED_GTIME) { -  string c = #" -  mapping(string:int) gtime = gmtime (timestamp);"; -  a_func += c, e_func += c; -  } -  -  if (log_flags & LOG_NEED_COOKIES) { -  a_func += #" -  request_id->init_cookies();"; -  } -  -  a_func += sprintf(#" -  string data = sprintf( %O%{, -  %s%} );", a_format, a_args ); -  e_func += sprintf(#" -  string data = sprintf( %O%{, -  %s%} );", e_format, e_args ); -  -  if (log_flags & LOG_ASYNC_HOST) -  { -  a_func += #" -  roxen.ip_to_host(request_id->remoteaddr,do_async_write, -  data, request_id->remoteaddr, callback );"; -  } else { -  a_func += #" -  callback( data );"; -  } -  a_func += #" -  } - "; -  -  e_func += #" -  callback (data); -  } - "; -  -  string src = #" -  inherit ___LogFormat;" + a_func + e_func; -  program res; -  if (mixed err = catch (res = compile_string (src))) { -  werror ("Failed to compile program: %s\n", src); -  throw (err); -  } -  mixed err = catch { -  string enc = encode_value(res, master()->Encoder (res)); -  -  con->query("REPLACE INTO compiled_formats (md5,full,enc) VALUES (%s,%s,%s)", -  kmd5, fmt, enc); -  }; -  if (err) { -  master()->handle_error(err); -  } -  -  LogFormat lf = res(); -  compiled_log_access[fmt] = lf->log_access; -  compiled_log_event[fmt] = lf->log_event; -  return lf; - } -  -  - // Security patterns -  - //! This array contains the compilation information for the different - //! security checks for e.g. @tt{htaccess@}. The layout of the top array is - //! a quadruple of sscanf string that the security command should match, - //! the number of arguments that it takes, an array with the actual - //! compilation information, and a symbol identifying the class of tests - //! the test belongs to. - //! - //! @array - //! @elem string command_sscanf_string - //! String to be passed as second argument to @[array_sscanf()] - //! to perform the match for the pattern. - //! @elem int(0..) number_of_arguments - //! Number of elements expected in the array returned by - //! @[array_sscanf()] for a proper match. - //! @elem array(function|string|int|multiset) actual_tests - //! In the tests array the following types has the following meaning: - //! @mixed - //! @type function - //! The function will be run during compilation. It gets the values - //! acquired through sscanf-ing the command as input and should return - //! an array with corresponding data. - //! @type string - //! The string will be compiled into the actual test code. It is - //! first modified as - //! @expr{str = sprintf(str, @@args)@} - //! where args are the arguments from the command after it has been - //! processed by the provided function, if any. - //! @type multiset - //! Strings in a multiset will be added before the string above. - //! should typically be used for variable declarations. - //! @type int - //! Signals that an authentication request should be sent to the user - //! upon failure. - //! @endmixed - //! @elem string state_symbol_string - //! Used to group the results from a class of tests. - //! Currently the following values are used: - //! @string - //! @value "ip" - //! @value "user" - //! @value "group" - //! @value "time" - //! @value "referer" - //! @value "day" - //! @value "language" - //! @value "luck" - //! @endstring - //! @endarray - //! - //! @note - //! It's up to the security checks in this file to ensure that - //! nothing is overcached. All patterns that perform checks using - //! information from the client (such as remote address, referer etc) - //! @b{have@} to use @[RequestID()->register_vary_callback()] (preferred), - //! or @[NOCACHE()] or @[NO_PROTO_CACHE()]. It's not necessary, however, - //! to do this for checks that use the authentication module API, since - //! then it's up to the user database and authentication modules to ensure - //! that nothing is overcached. - //! - //! @seealso - //! @[RequestID()->register_vary_callback()], @[NOCACHE()], - //! @[NO_PROTO_CACHE()], @[array_sscanf()] - array(array(string|int|array)) security_checks = ({ -  ({ "ip=%s", 1, ({ -  lambda(string x) { -  mapping(int:array(int)) ip_masks = ([]); -  array(string) globs = ({}); -  string ret; -  foreach(x/",", string ip_mask) { -  if (sscanf(ip_mask, "%s:%s", string ip, string mask) == 2) { -  int m = Roxen.ip_to_int(mask); -  if (m & 0x80000000) m -= 0x100000000; -  ip_masks[m] += ({ Roxen.ip_to_int(ip) }); -  } else if (sscanf(ip_mask, "%s/%d", string ip, int mask) == 2) { -  mask = -1 - (0xffffffff >> mask); -  ip_masks[mask] += ({ Roxen.ip_to_int(ip) }); -  } else { -  globs += ({ ip_mask }); -  } -  } -  if (sizeof(ip_masks)) { -  foreach(ip_masks; int mask; array(int) ip) { -  if (!mask) continue; -  if (ret) ret += " ||\n "; -  else ret = ""; -  if (sizeof(ip) == 1) { -  ret += -  sprintf("((remote_ip & ~0x%08x) == 0x%08x)", -  ~mask, ip[0] & mask); -  } else { -  ret += -  sprintf("(<%{0x%08x,%}>)[remote_ip & ~0x%08x]", -  map(ip, `&, mask), ~mask); -  } -  } -  } -  foreach(globs, string glob) { -  if (ret) ret += " ||\n "; -  else ret = ""; -  ret += sprintf("glob(%O, id->remoteaddr)", glob); -  } -  return ({ -  ret, -  }); -  }, - #if defined(SECURITY_PATTERN_DEBUG) || defined(HTACCESS_DEBUG) -  " report_debug(sprintf(\"Verifying against IP %%O (0x%%08x).\\n\",\n" -  " id->remoteaddr, remote_ip));\n" - #endif /* SECURITY_PATTERN_DEBUG || HTACCESS_DEBUG */ -  " if (%s)", -  (< " int remote_ip = Roxen.ip_to_int(id->remoteaddr)" >), -  }), "ip", }), -  ({ "user=%s",1,({ 1, -  lambda( string x ) { -  return ({sprintf("((multiset)(< %{%O, %}>))", x/"," )}); -  }, -  -  " if (((user || (user = authmethod->authenticate(id, userdb_module)))\n" -  " && ((%[0]s->any) || (%[0]s[user->name()]))) || %[0]s->ANY) ", -  (<" User user" >), -  // No need to NOCACHE () here, since it's up to the -  // auth-modules to do that. -  }), "user", }), -  ({ "group=%s",1,({ 1, -  lambda( string x ) { -  return ({sprintf("((multiset)(< %{%O, %}>))", x/"," )}); -  }, -  " if ((user || (user = authmethod->authenticate(id, userdb_module)))\n" -  " && ((%[0]s->any && sizeof(user->groups())) ||\n" -  " sizeof(mkmultiset(user->groups())&%[0]s)))", -  (<" User user" >), -  // No need to NOCACHE () here, since it's up to the -  // auth-modules to do that. -  }), "group", }), -  ({ "dns=%s",1,({ -  " if(!dns && \n" -  " ((dns=roxen.quick_ip_to_host(id->remoteaddr))==id->remoteaddr))\n" -  " if( (id->misc->delayed+=0.1) < 1.0 )\n" -  " return Roxen.http_try_again( 0.1 );\n" -  " if (sizeof(filter(%[0]O/\",\",\n" -  " lambda(string q){return glob(lower_case(q),lower_case(dns));})))", -  (< " string dns" >), -  }), "ip", }), -  ({ "time=%d:%d-%d:%d",4,({ -  (< " mapping l = localtime(time(1))" >), -  (< " int th = l->hour, tm = l->min" >), -  // No need to NOCACHE() here, does not depend on client. -  " if (((th >= %[0]d) && (tm >= %[1]d)) &&\n" -  " ((th <= %[2]d) && (tm <= %[3]d)))", -  }), "time", }), -  ({ "referer=%s", 1, ({ -  (< -  " string referer = sizeof(id->referer||({}))?id->referer[0]:\"\"; " -  >), -  " if( sizeof(filter(%[0]O/\",\",\n" -  " lambda(string q){return glob(q,referer);})))", -  }), "referer", }), -  ({ "day=%s",1,({ -  lambda( string q ) { -  multiset res = (<>); -  foreach( q/",", string w ) if( (int)w ) -  res[((int)w % 7)] = 1; -  else -  res[ (["monday":1,"thuesday":2,"wednesday":3,"thursday":4,"friday":5, -  "saturday":6,"sunday":0,"mon":1, "thu":2, "wed":3, "thu":4, -  "fri":5, "sat":6, "sun":0, ])[ lower_case(w) ] ] = 1; -  return ({sprintf("(< %{%O, %}>)", (array)res)}); -  }, -  (< " mapping l = localtime(time(1))" >), -  // No need to NOCACHE() here, does not depend on client. -  " if (%[0]s[l->wday])" -  }), "day", }), -  ({ "accept_language=%s",1,({ -  " if (has_value(id->misc->pref_languages->get_languages(), %O))", -  (<" NO_PROTO_CACHE()" >), -  }), "language", }), -  ({ "luck=%d%%",1,({ -  lambda(int luck) { return ({ 100-luck }); }, -  // Not really any need to NOCACHE() here, since it does not depend -  // on client. However, it's supposed to be totally random. -  " if( random(100)<%d )", -  (<" NOCACHE()" >), -  }), "luck", }), - }); -  - #define DENY 0 - #define ALLOW 1 -  - function(RequestID:mapping|int) compile_security_pattern( string pattern, -  RoxenModule m ) - //! Parse a security pattern and return a function that when called - //! will do the checks required by the format. - //! - //! The syntax is: - //! - //! userdb userdatabase module - //! authmethod authentication module - //! realm realm name - //! - //! Below, CMD is one of 'allow' and 'deny' - //! - //! CMD ip=ip/bits[,ip/bits] [return] - //! CMD ip=ip:mask[,ip:mask] [return] - //! CMD ip=pattern [return] - //! - //! CMD user=name[,name,...] [return] - //! CMD group=name[,name,...] [return] - //! - //! CMD dns=pattern [return] - //! - //! CMD day=pattern [return] - //! - //! CMD time=<start>-<stop> [return] - //! times in HH:mm format - //! - //! CMD referer=pattern [return] - //! Check the referer header. - //! - //! CMD accept_language=language [return] - //! - //! CMD luck=entry_chance% [return] - //! Defines the minimum level of luck required. All attempts - //! gets accepted at 0%, and no attempts gets accepted at 100%. - //! - //! pattern is a glob pattern. - //! - //! return means that reaching this command results in immediate - //! return, only useful for 'allow'. - //! - //! 'deny' always implies a return, no futher testing is done if a - //! 'deny' match. - { -  // Now, this cache is not really all that performance critical, I -  // mostly coded it as a proof-of-concept, and because it was more -  // fun that trying to find the bug in the image-cache at the moment. -  -  // Note similar code in compile_log_format. -  if (pattern == "") -  return 0; -  string kmd5 = md5( pattern ); -  - #if !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG) -  array tmp = -  dbm_cached_get( "local" ) -  ->query("SELECT full,enc FROM compiled_formats WHERE md5=%s", kmd5 ); -  -  if( sizeof(tmp) && (tmp[0]->full == pattern) ) -  { -  mixed err = catch { -  return decode_value( tmp[0]->enc, master()->Decoder() )()->f; -  }; - // #ifdef DEBUG -  if (describe_error (err) != -  "Cannot decode programs encoded with other pike version.\n") -  report_warning ("Decoding of dumped security pattern failed " -  "(will recompile):\n%s", describe_backtrace(err)); - // #endif -  } - #endif /* !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG) */ -  -  -  string code = ""; -  array variables = ({ " object userdb_module", -  " object authmethod = id->conf", -  " string realm = \"User\"", -  " mapping(string:int|mapping) state = ([])", -  " id->register_vary_callback(0, vary_cb)", -  }); -  -  // Some state variables for optimizing. -  int all_shorted = 1; // All allow patterns have return. -  int need_auth = 0; // We need auth for some checks. -  int max_short_code = 0; // Max fail code for return checks. -  int patterns; // Number of patterns. -  multiset(string) checks = (<>); // Checks in state. -  -  foreach( pattern / "\n", string line ) -  { -  line = String.trim_all_whites( line ); -  if( !strlen(line) || line[0] == '#' ) -  continue; -  sscanf( line, "%[^#]#", line ); -  -  int cmd; -  -  if( sscanf( line, "allow %s", line ) ) -  cmd = ALLOW; -  else if( sscanf( line, "deny %s", line ) ) -  cmd = DENY; -  else if( sscanf( line, "userdb %s", line ) ) -  { -  line = String.trim_all_whites( line ); -  if( line == "config_userdb" ) -  code += " userdb_module = roxen.config_userdb_module;\n"; -  else if( line == "all" ) -  code += " userdb_module = 0;\n"; -  else if( !m->my_configuration()->find_user_database( line ) ) -  m->report_notice( LOC_M( 58,"Syntax error in security patterns: " -  "Cannot find the user database '%s'")+"'\n", -  line); -  else -  code += -  sprintf(" userdb_module = id->conf->find_user_database( %O );\n", -  line); -  continue; -  } -  else if( sscanf( line, "authmethod %s", line ) ) -  { -  line = String.trim_all_whites( line ); -  if( line == "all" ) -  code += " authmethod = id->conf;\n"; -  else if( !m->my_configuration()->find_auth_module( line ) ) -  m->report_notice( LOC_M( 59,"Syntax error in security patterns: " -  "Cannot find the auth method '%s'")+"\n", -  line); -  else -  code += -  sprintf(" authmethod = id->conf->find_auth_module( %O );\n", -  line); -  continue; -  } -  else if( sscanf( line, "realm %s", line ) ) -  { -  line = String.trim_all_whites( line ); -  code += sprintf( " realm = %O;\n", line ); -  continue; -  } -  else { -  m->report_notice( LOC_M( 60,"Syntax error in security patterns: " -  "Expected 'allow' or 'deny'\n" )); -  continue; -  } -  int shorted = sscanf( line, "%s return", line ); -  -  -  // Notes on the variables @[state] and @[short_fail]: -  // -  // The variable @[state] has several potential entries -  // (currently "ip", "user", "group", "time", "date", -  // "referer", "language" and "luck"). -  // An entry exists in the mapping if a corresponding accept directive -  // has been executed. -  // -  // The variable @[short_fail] contains a non-zero entry if -  // a potentially acceptable accept with return has been -  // encountered. -  // -  // Valid values in @[state] and @[short_fail] are: -  // @int -  // @value 0 -  // Successful match. -  // @value 1 -  // Plain failure. -  // @value 2 -  // Fail with authenticate. -  // @endint -  // -  // Shorted directives will only be regarded if all unshorted directives -  // encountered at that point have succeeded. -  // If the checking ends with an ok return unshorted directives of -  // the same class will be disregarded as well as any potential -  // short directives. -  // If there are unshorted directives of type 2 and none of type 1, -  // an auth request will be sent. -  // If there are unshorted directives, and all of them have been -  // satisfied an OK will be sent. -  // If there is a potential short directive of type 2, an auth -  // request will be sent. -  // If there are no unshorted directives and no potential short -  // directives an OK will be sent. -  // Otherwise a FAIL will be sent. -  -  foreach(security_checks, array(string|int|array) check) -  { -  array args; -  if (sizeof(args = array_sscanf(line, check[0])) == check[1]) -  { -  // Got a match for this security check. -  patterns++; -  int thr_code = 1; -  // run instructions. -  foreach(check[2], mixed instr ) -  { -  if( functionp( instr ) ) -  args = instr( @args ); -  else if( multisetp( instr ) ) -  { -  foreach( (array)instr, string v ) -  if( !has_value( variables, v ) ) -  variables += ({ v }); -  } -  else if( intp( instr ) ) { -  thr_code = 2; -  need_auth = 1; -  } -  else if( stringp( instr ) ) -  { -  code += sprintf( instr, @args )+"\n"; -  if( cmd == DENY ) -  { -  // Make sure we fail regardless of the setting -  // of all_shorted when we're done. -  if (thr_code < max_short_code) { -  code += sprintf(" {\n" -  " state->%s = %d;\n" -  " if (short_fail < %d)\n" -  " short_fail = %d;\n" -  " break;\n" -  " }\n", -  check[3], thr_code, -  thr_code, -  thr_code); -  } else { -  code += sprintf(" {\n" -  " state->%s = %d;\n" -  " short_fail = %d;\n" -  " break;\n" -  " }\n", -  check[3], thr_code, -  thr_code); -  max_short_code = thr_code; -  } -  } -  else -  { -  if (shorted) { -  // OK with return. Ignore FAIL/return. -  if (all_shorted) { -  code += - #if defined(SECURITY_PATTERN_DEBUG) || defined(HTACCESS_DEBUG) -  " {\n" -  " report_debug(\" Result: 0 (fast return)\\n\");\n" -  " return 0;\n" -  " }\n"; - #else /* !SECURITY_PATTERN_DEBUG && !HTACCESS_DEBUG */ -  " return 0;\n"; - #endif /* SECURITY_PATTERN_DEBUG || HTACCESS_DEBUG */ -  } else { -  code += " {\n"; -  if (checks[check[3]]) { -  code += -  sprintf(" m_delete(state, %O);\n", -  check[3]); -  } -  if (max_short_code) { -  code += " short_fail = 0;\n"; -  } -  code += -  " break;\n" -  " }\n"; -  } -  // Handle the fail case. -  if (sizeof(checks)) { -  // Check that we can satify all preceeding tests. -  code += -  " else if (!sizeof(filter(values(state),\n" -  " values(state))))\n"; -  } else { -  code += " else\n"; -  } -  // OK so far for non return tests. -  if (thr_code < max_short_code) { -  code += -  sprintf(" if (short_fail < %d)\n" -  " short_fail = %d;\n", -  thr_code, -  thr_code); -  } else { -  code += -  sprintf(" short_fail = %d;\n", thr_code); -  max_short_code = thr_code; -  } -  } else { -  // OK without return. Mark as OK. -  code += -  sprintf(" {\n" -  " state->%s = 0;\n" -  " }\n", -  check[3]); -  all_shorted = 0; -  // Handle the fail case. -  if (checks[check[3]]) { -  // If not marked -  // set the failure level. -  code += -  sprintf(" else if (zero_type(state->%s))\n", -  check[3]); -  } else { -  code += " else\n"; -  } -  code += sprintf(" {\n" -  " state->%s = %d;\n" -  " }\n", -  check[3], thr_code); -  checks[check[3]] = 1; -  } -  } -  } -  } -  break; -  } -  } -  } -  if( !patterns ) return 0; -  code = (" do {\n" + -  code + -  " } while(0);\n"); -  code = ("#include <module.h>\n" -  "int|mapping f( RequestID id )\n" -  "{\n" + -  (variables * ";\n") + -  ";\n" + -  (max_short_code?" int short_fail;\n":"") + - #if defined(SECURITY_PATTERN_DEBUG) || defined(HTACCESS_DEBUG) -  sprintf(" report_debug(\"Verifying against pattern:\\n\"\n" -  "%{ \" \" %O \"\\n\"\n%}" -  " \"...\\n\");\n" -  "%s"+ -  (max_short_code? -  " report_debug(sprintf(\" Short code: %%O\\n\",\n" -  " short_fail));\n":"")+ -  " report_debug(sprintf(\" Result: %%O\\n\",\n" -  " state));\n", -  pattern/"\n", code) + - #else /* !SECURITY_PATTERN_DEBUG && !HTACCESS_DEBUG */ -  code + - #endif /* SECURITY_PATTERN_DEBUG || HTACCESS_DEBUG */ -  -  (!all_shorted? -  " int fail = 0;\n" -  " foreach(values(state), int value) {\n" -  " fail |= value;\n" -  " }\n" -  " if (!fail)\n" -  " return 0;\n": -  "") + -  (max_short_code > 1? -  " if (short_fail > 1)\n" -  " return authmethod->authenticate_throw(id, realm);\n": -  "") + -  (!all_shorted && need_auth? -  " if (fail == 2)\n" -  " return authmethod->authenticate_throw(id, realm);\n": -  "") + -  " return 1;\n" -  "}\n" -  "string vary_cb(string ignored, RequestID id)\n" -  "{\n" -  " int|mapping res = f(id);\n" -  " if (intp(res)) return (string) res;\n" -  " return 0; // FIXME: Analyze the mapping.\n" -  "}\n"); - #if defined(SECURITY_PATTERN_DEBUG) || defined(HTACCESS_DEBUG) -  report_debug(sprintf("Compiling security pattern:\n" -  "%{ %s\n%}\n" -  "Code:\n" -  "%{ %s\n%}\n", -  pattern/"\n", -  code/"\n")); - #endif /* SECURITY_PATTERN_DEBUG || HTACCESS_DEBUG */ -  mixed res = compile_string( code ); -  - #if !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG) -  dbm_cached_get( "local" ) -  ->query("REPLACE INTO compiled_formats (md5,full,enc) VALUES (%s,%s,%s)", -  kmd5,pattern,encode_value( res, master()->Encoder (res) ) ); - #endif /* !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG) */ -  -  return compile_string(code)()->f; - } -  -  - protected string cached_hostname = gethostname(); -  - class LogFile - { -  public string fname; // Was public before... -  public string compressor_program; // Was public before... -  private Thread.Mutex lock = Thread.Mutex(); -  private Stdio.File fd; -  private int opened; -  private bool compressor_exists; -  private bool auto_file_removal; -  private int days_to_keep_files; -  -  protected void create(string fname, -  string|void compressor_program, -  int|void days_to_keep_files) -  { -  this::fname = fname; -  this::compressor_program = compressor_program; -  this::days_to_keep_files = days_to_keep_files; -  compressor_exists = compressor_program && sizeof(compressor_program); -  auto_file_removal = days_to_keep_files && days_to_keep_files > 0; -  } -  -  // FIXME: compress_logs is limited to scanning files with filename -  // substitutions within a fixed directory (e.g. -  // "$LOGDIR/test/Log.%y-%m-%d", not "$LOGDIR/test/%y/Log.%m-%d"). -  private Process.Process compressor_process; -  private int last_scan_time; -  -  //! Also deletes old files. -  //! -  // Will not scan for files if compressor is running. This means we might not -  // remove an old file because the compressor is running but that does not -  // matter since this function is ran so often. Sooner or later files will be -  // compressed (if there is a compressor) and old files will be deleted (if -  // days_to_keep_files > 0). -  private void compress_logs(string fname, string active_log) -  { -  if(!compressor_exists && !auto_file_removal) -  // No compressor program specified, nor is auto file removal active... -  return; -  if(compressor_process && !compressor_process->status()) -  return; // The compressor is running... -  if(time(1) - last_scan_time < 300) -  return; // Scan for files at most once every 5 minutes... -  last_scan_time = time(1); -  fname = roxen_path(fname); -  active_log = roxen_path(active_log); -  string dir = dirname(fname); -  int min_mtime = time(1) - (days_to_keep_files * 24 * 60 * 60); -  string pattern = "^"+replace(basename(fname), -  ({ "%y", "%m", "%d", "%h", "%H" }), -  ({ "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", -  "[0-9][0-9]", "[0-9][0-9]", "(.+)" })); -  Regexp regexp = Regexp(pattern); -  Regexp regexp_non_compressed = Regexp(pattern + "$"); -  foreach(sort(get_dir(dir) || ({})), string filename_candidate) -  { -  if(filename_candidate == basename(active_log)) -  { -  continue; // Don't try to compress the active log just yet... -  } -  else if(compressor_exists && -  regexp_non_compressed->match(filename_candidate)) -  { -  string compress_file = combine_path(dir, filename_candidate); -  Stdio.Stat stat = file_stat(compress_file); -  if(!stat || time(1) < stat->mtime + 1200) -  continue; // Wait at least 20 minutes before compressing log file... -  werror("Compressing log file %O\n", compress_file); -  compressor_process = Process.Process(({ compressor_program, -  compress_file })); -  return; -  } -  else if(auto_file_removal && regexp->match(filename_candidate)) -  { -  // Wipe the file if it is old. -  string log_file = combine_path(dir, filename_candidate); -  Stdio.Stat stat = file_stat(log_file, 1); // 1 means symlinks will not be followed. -  if(stat->isreg && stat->mtime < min_mtime) -  { -  werror("Deleting log file %O due to old age.\n", log_file); -  rm(log_file); -  } -  } -  } -  } -  -  private void do_open_co() { handle(do_open); } -  private void do_open(void|object mutex_key) -  { -  if (!mutex_key) mutex_key = lock->lock(); -  -  mixed parent; -  if (catch { parent = function_object(object_program(this_object())); } || -  !parent) { -  // Our parent (aka the configuration) has been destructed. -  // Time to die. -  remove_call_out(do_open_co); -  remove_call_out(do_close_co); -  destruct(); -  return; -  } -  string ff = fname; -  mapping m = localtime(time(1)); -  m->year += 1900; // Adjust for years being counted since 1900 -  m->mon++; // Adjust for months being counted 0-11 -  if(m->mon < 10) m->mon = "0"+m->mon; -  if(m->mday < 10) m->mday = "0"+m->mday; -  if(m->hour < 10) m->hour = "0"+m->hour; -  ff = replace(fname,({"%d","%m","%y","%h", "%H" }), -  ({ (string)m->mday, (string)(m->mon), -  (string)(m->year),(string)m->hour, -  cached_hostname, -  })); -  compress_logs(fname, ff); -  mkdirhier( ff ); -  fd = open( ff, "wac" ); -  if(!fd) -  { -  remove_call_out(do_open_co); -  call_out(do_open_co, 120); -  report_error(LOC_M(37, "Failed to open logfile")+" "+fname+" " - #if constant(strerror) -  "(" + strerror(errno()) + ")" - #endif -  "\n"); -  return; -  } -  opened = 1; -  remove_call_out(do_open_co); -  call_out(do_open_co, 900); -  remove_call_out(do_close_co); -  call_out(do_close_co, 10.0); -  } -  -  private void do_close_co() { handle(close); } -  -  void close() -  { -  object mutex_key = lock->lock(); -  -  destruct( fd ); -  opened = 0; -  } -  -  private array(string) write_buf = ({}); -  private void do_the_write() -  { -  object mutex_key = lock->lock(); -  -  if (!opened) do_open(mutex_key); -  if (!opened) return; -  if (!sizeof (write_buf)) return; -  -  array(string) buf = write_buf; -  // Relying on the interpreter lock here. -  write_buf = ({}); -  -  mixed err = catch (fd->write(buf)); -  if (err) -  catch { -  foreach (write_buf, string str) -  if (String.width (str) > 8) -  werror ("Got wide string in log output: %O\n", str); -  }; -  -  remove_call_out(do_close_co); -  call_out(do_close_co, 10.0); -  -  if (err) -  throw (err); -  } -  -  int write( string what ) -  { -  write_buf += ({ what }); -  if (sizeof (write_buf) == 1) -  background_run (1, do_the_write); -  return strlen(what); -  } - } +    Newline at end of file removed.