Roxen.git / server / base_server / roxen.pike

version» Context lines:

Roxen.git/server/base_server/roxen.pike:94:   string md5( string what )   {    return Gmp.mpz(Crypto.MD5.hash( what ),256)->digits(32);   }      string query_configuration_dir()   {    return configuration_dir;   }    + //! @ignore + array(string) query_hot_reload_modules() + //! Returns an array of modules added for hot reloading via + //! @tt{--module-hot-reload=<modname>@}. + { +  if (hot_reload_modules) { +  return map(replace(hot_reload_modules, " ", ",")/",", +  String.trim_all_whites) - ({ "" }); +  } +  +  return ({}); + } +  + array(string) query_hot_reload_modules_conf() + //! Returns an array of modules added for hot reloading via + //! @tt{--module-hot-reload-conf=<conf>@}. + { +  if (hot_reload_modules_conf) { +  return map(replace(hot_reload_modules_conf, " ", ",")/",", +  String.trim_all_whites) - ({ "" }); +  } +  +  return 0; + } + //! @endignore +    array(string|int) filename_2 (program|object o)   {    if( objectp( o ) )    o = object_program( o );       string fname = Program.defined (o);    int line;    if (fname) {    array(string) p = fname / ":";    if (sizeof (p) > 1 && p[-1] != "" && sscanf (p[-1], "%d%*c", int l) == 1) {
Roxen.git/server/base_server/roxen.pike:131:    return ({fname, line});   }      string filename( program|object o )   {    [string fname, int line] = filename_2 (o);    return fname || "(unknown program)";   }      protected int once_mode; + // String of modules added for hot reloading via --module-hot-reload=<mod> + protected string hot_reload_modules; + protected string hot_reload_modules_conf;      // Note that 2.5 is a nonexisting version. It's only used for the   // cache static optimization for tags such as <if> and <emit> inside   // <cache> since that optimization can give tricky incompatibilities   // with 2.4.   // Note also that 5.3 only existed in the Print repository, and   // thus is skipped here.   array(string) compat_levels = ({"2.1", "2.2", "2.4", "2.5",    "3.3", "3.4",    "4.0", "4.5",    "5.0", "5.1", "5.2", "5.4", "5.5",    "6.0", "6.1",   });    -  + // Compat stubs for relocated methods   #ifdef THREADS - mapping(string:string) thread_names = ([]); -  +    string thread_name_from_addr(string hex_addr)   { -  // Lookup using a key like "Thread.Thread(0x...)" that matches what -  // sprint("%O") generates. -  string th_key = "Thread.Thread(" + hex_addr + ")"; -  return thread_names[th_key]; +  return Roxen.thread_name_from_addr(hex_addr);   }    - string thread_name( object thread, int|void skip_auto_name ) + string thread_name(object thread, int|void skip_auto_name)   { -  string tn; -  if( thread_names[ tn=sprintf("%O",thread) ] || skip_auto_name ) -  return thread_names[tn]; -  return tn; +  return Roxen.thread_name(thread, skip_auto_name);   }      void name_thread( object thread, string name )   { -  string th_key = sprintf("%O", thread); -  if (name) -  thread_names[th_key] = name; -  else -  m_delete(thread_names, th_key); +  Roxen.name_thread(thread, name);   } -  + #endif    - #endif /* THREADS */ +       /* Used by read_config.pike, since there seems to be problems with    * overloading otherwise.    */   protected Privs PRIVS(string r, int|string|void u, int|string|void g)   {    return Privs(r, u, g);   }      // Current Configuration.
Roxen.git/server/base_server/roxen.pike:295:   #endif /* THREADS */    roxenloader.real_exit(exit_code);    }    if (shutdown_recurse++) return;      #ifndef NO_SLOW_REQ_BT    // Turn off the backend thread monitor while we're shutting down.    slow_be_timeout_changed();   #endif    +  DBManager.stop_backup_thread(); +     if ((apply_patches || query("patch_on_restart")) > 0) {    mixed err = catch {    foreach(plib->file_list_imported(), mapping(string:mixed) item) {    report_notice("Applying patch %s...\n", item->metadata->id);    mixed err = catch {    plib->install_patch(item->metadata->id,    "Internal Administrator");    };    if (err) {    report_error("Failed to install patch %s: %s\n",
Roxen.git/server/base_server/roxen.pike:371:   /*    * handle() stuff    */      #ifdef THREADS   // function handle = threaded_handle;      Thread.Thread do_thread_create(string id, function f, mixed ... args)   {    Thread.Thread t = thread_create(f, @args); -  name_thread( t, id ); +  Roxen.name_thread( t, id );    return t;   }      #if 1   constant Queue = Thread.Queue;   #else   // Shamelessly uses facts about pikes preemting algorithm.   // Might have to be fixed in the future.   class Queue   //! Thread.Queue lookalike, which uses some archaic and less
Roxen.git/server/base_server/roxen.pike:454: Inside #if undefined(NO_SLOW_REQ_BT)
  // This is a system to dump all threads whenever a request takes   // longer than a configurable timeout.      protected Pike.Backend slow_req_monitor; // Set iff slow req bt is enabled.   protected float slow_req_timeout, slow_be_timeout;      protected void slow_req_monitor_thread (Pike.Backend my_monitor)   {    // my_monitor is just a safeguard to ensure we don't get multiple    // monitor threads. -  name_thread(this_thread(), "Slow request monitor"); +  Roxen.name_thread(this_thread(), "Slow Request Monitor");    while (slow_req_monitor == my_monitor)    slow_req_monitor (3600.0); -  name_thread(this_thread(), 0); +  Roxen.name_thread(this_thread(), 0);   }      protected mixed slow_be_call_out;      protected void slow_be_before_cb()   {   #ifdef DEBUG    if (this_thread() != backend_thread) error ("Run from wrong thread.\n");   #endif    if (Pike.Backend monitor = slow_be_call_out && slow_req_monitor) {
Roxen.git/server/base_server/roxen.pike:580: Inside #if undefined(NO_SLOW_REQ_BT)
      if (thread == backend_thread && !slow_be_call_out) {    // Avoid false alarms for the backend thread if we got here due to    // a race. Should perhaps have something like this for the handler    // threads too, but otoh races are more rare there due to the    // longer timeouts.    }       else {    string th_name = -  ((thread != backend_thread) && thread_name(thread, 1)) || ""; +  ((thread != backend_thread) && Roxen.thread_name(thread, 1)) || "";    if (sizeof(th_name))    th_name = " - " + th_name + " -";    report_debug ("###### %s 0x%x%s has been busy for more than %g seconds.\n",    thread == backend_thread ? "Backend thread" : "Thread",    thread->id_number(), th_name, timeout);    int hrnow = gethrtime();    if ((hrnow - last_dump_hrtime) / 1E6 < slow_req_timeout / 2) {    describe_thread (thread);    } else {    last_dump_hrtime = hrnow;
Roxen.git/server/base_server/roxen.pike:609: Inside #if undefined(NO_SLOW_REQ_BT)
  }      protected void report_slow_thread_finished (Thread.Thread thread,    float time_spent)   {    if (query ("slow_req_bt_count") == 0) {    return;    }       string th_name = -  ((thread != backend_thread) && thread_name(thread, 1)) || ""; +  ((thread != backend_thread) && Roxen.thread_name(thread, 1)) || "";    if (sizeof(th_name))    th_name = " - " + th_name + " -";       report_debug ("###### %s 0x%x%s finished after %.2f seconds.\n",    (thread == backend_thread ?    "Backend thread" : "Thread"),    thread->id_number(), th_name, time_spent);   }      #endif // !NO_SLOW_REQ_BT
Roxen.git/server/base_server/roxen.pike:847:   {    if (query("numthreads") <= 1) {    set( "numthreads", 1 );    report_warning (LOC_S(1, "Starting one thread to handle requests.")+"\n");    } else {    report_notice (LOC_S(2, "Starting %d threads to handle requests.")+"\n",    query("numthreads") );    }    array(object) new_threads = ({});    for(; number_of_threads < query("numthreads"); number_of_threads++) -  new_threads += ({ do_thread_create( "Handle thread [" + +  new_threads += ({ do_thread_create( "Handle Thread [" +    number_of_threads + "]",    handler_thread, number_of_threads ) });    handler_threads += new_threads;   }      protected int num_hold_messages;   protected Thread.Condition hold_wakeup_cond = Thread.Condition();   // Note: There are races in the use of this condition variable, but   // the only effect of that is that some handler thread might be   // considered hung when it's actually waiting on hold_wakeup_cond, and
Roxen.git/server/base_server/roxen.pike:920:       int blocked_threads = number_of_threads - threads_on_hold;    int threads_to_create = numthreads - threads_on_hold;       THREAD_WERR("Releasing " + threads_on_hold + " threads on hold");    cond->broadcast();       if (threads_to_create > 0) {    array(object) new_threads = ({});    for (int n = 0; n < threads_to_create; number_of_threads++, n++) -  new_threads += ({ do_thread_create( "Handle thread [" + +  new_threads += ({ do_thread_create( "Handle Thread [" +    number_of_threads + "]",    handler_thread, number_of_threads ) });    handler_threads += new_threads;    report_notice ("Created %d new handler threads to compensate "    "for %d blocked ones.\n", threads_to_create, blocked_threads);    }    }    else {    THREAD_WERR("Ignoring request to release handler threads during stop");    return;
Roxen.git/server/base_server/roxen.pike:6043: Inside #if constant(spider.shuffle)
  #if constant(spider.shuffle)    }   #endif   }      // Dump a single thread.   void describe_thread (Thread.Thread thread)   {    int hrnow = gethrtime();    string thread_descr = ""; -  if (string th_name = thread_name(thread, 1)) +  if (string th_name = Roxen.thread_name(thread, 1))    thread_descr += " - " + th_name;    if (int start_hrtime = thread_task_start_times[thread])    thread_descr += sprintf (" - busy for %.3fs",    (hrnow - start_hrtime) / 1e6);    report_debug(">> ### Thread 0x%x%s:\n",    thread->id_number(),    thread_descr);    // Use master()->describe_backtrace to sidestep the background    // failure wrapper that's active in RUN_SELF_TEST.    string th_bt = master()->describe_backtrace (thread->backtrace());
Roxen.git/server/base_server/roxen.pike:6065:    // Expand any occurrences of:    // Thread.Mutex(/*locked by 0x....*/)    // to:    // Thread.Mutex(/*locked by 0x.... - <thread name>*/)    string bt_separator = "Thread.Mutex(/*locked by ";    if (has_value(th_bt, bt_separator)) {    array(string) bt_segs = th_bt / bt_separator;    if (sizeof(bt_segs) > 1) {    foreach (bt_segs; int idx; string bt_seg) {    if (sscanf(bt_seg, "0x%[0-9a-fA-F]*/", string th_hex_addr)) { -  if (string th_name = thread_name_from_addr("0x" + th_hex_addr)) { +  if (string th_name = Roxen.thread_name_from_addr("0x" + th_hex_addr)) {    bt_segs[idx] =    "0x" + th_hex_addr + " - " + th_name +    bt_seg[sizeof(th_hex_addr) + 2..];    }    }    }    th_bt = bt_segs * bt_separator;    }    }   
Roxen.git/server/base_server/roxen.pike:6173:   constant cdt_poll_interval = 5; // Seconds.   constant cdt_dump_seq_interval = 60;      string cdt_directory, cdt_filename;      Thread.Thread cdt_thread;   int cdt_next_seq_dump;      void cdt_poll_file()   { -  name_thread(this_thread(), "Dump thread file monitor"); +  Roxen.name_thread(this_thread(), "Dump Thread File Monitor");    while (this && query ("dump_threads_by_file")) {    if (array(string) dir = r_get_dir (cdt_directory)) {    if (has_value (dir, cdt_filename)) {    r_rm (cdt_directory + "/" + cdt_filename);    describe_all_threads();    }    else if (time() >= cdt_next_seq_dump) {    dir = glob (cdt_filename + ".*", dir);    if (sizeof (dir)) {    string file = dir[0];
Roxen.git/server/base_server/roxen.pike:6197:    if (--count > 0) {    open (cdt_directory + "/" + cdt_filename + "." + count,    "cwt");    cdt_next_seq_dump = time (1) + cdt_dump_seq_interval;    }    }    }    }    sleep (cdt_poll_interval);    } -  name_thread(this_thread(), 0); +  Roxen.name_thread(this_thread(), 0);    cdt_thread = 0;   }      void cdt_changed (Variable.Variable v)   {    if (cdt_directory && v->query() && !cdt_thread)    cdt_thread = Thread.thread_create (cdt_poll_file);   }      // ----------------------------------------
Roxen.git/server/base_server/roxen.pike:6668:    DDUMP( "base_server/wizard.pike" );    DDUMP( "base_server/proxyauth.pike" );    DDUMP( "base_server/module.pike" );    DDUMP( "base_server/throttler.pike" );       mark_fd(0, "Stdin");    mark_fd(1, "Stdout");    mark_fd(2, "Stderr");       once_mode = (int)Getopt.find_option(argv, "o", "once"); +  hot_reload_modules = Getopt.find_option(argv, 0, "module-hot-reload"); +  hot_reload_modules_conf = Getopt.find_option(argv, 0, "module-hot-reload-conf");       configuration_dir =    Getopt.find_option(argv, "d",({"config-dir","configuration-directory" }),    ({ "ROXEN_CONFIGDIR", "CONFIGURATIONS" }), "../configurations");       if(configuration_dir[-1] != '/')    configuration_dir += "/";       restore_global_variables(); // restore settings...   
Roxen.git/server/base_server/roxen.pike:6744: Inside #if defined(SNMP_AGENT)
   snmpagent->enable();    report_debug("\benabled.\n");    snmpagent->start_trap();       } else    report_debug("\bdisabled.\n");   #endif // SNMP_AGENT       backend_thread = this_thread();   #ifdef THREADS -  name_thread( backend_thread, "Backend" ); +  Roxen.name_thread( backend_thread, "Backend" );   #else    report_debug("\n"    "WARNING: Threads not enabled!\n"    "\n");   #endif /* THREADS */       foreach(({ "testca.pem", "demo_certificate.pem" }), string file_name) {    if (!sizeof(roxenloader.package_directories)) break;    CertDB.register_pem_file(file_name);    string cert;
Roxen.git/server/base_server/roxen.pike:8200: Inside #if !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG)
   ->query("REPLACE INTO compiled_formats (md5,full,enc) VALUES (%s,%s,%s)",    kmd5,pattern,encode_value( res, master()->Encoder (res) ) );   #endif /* !defined(HTACCESS_DEBUG) && !defined(SECURITY_PATTERN_DEBUG) */       return compile_string(code)()->f;   }         protected string cached_hostname = gethostname();    - class LogFile(string fname, string|void compressor_program) + class LogFile   { -  +  public string fname; // Was public before... +  public string compressor_program; // Was public before...    private Thread.Mutex lock = Thread.Mutex();    private Stdio.File fd;    private int opened; -  +  private bool compressor_exists; +  private bool auto_file_removal; +  private int days_to_keep_files;    -  +  protected void create(string fname, +  string|void compressor_program, +  int|void days_to_keep_files) +  { +  this::fname = fname; +  this::compressor_program = compressor_program; +  this::days_to_keep_files = days_to_keep_files; +  compressor_exists = compressor_program && sizeof(compressor_program); +  auto_file_removal = days_to_keep_files && days_to_keep_files > 0; +  } +     // FIXME: compress_logs is limited to scanning files with filename    // substitutions within a fixed directory (e.g.    // "$LOGDIR/test/Log.%y-%m-%d", not "$LOGDIR/test/%y/Log.%m-%d").    private Process.Process compressor_process; -  private int last_compressor_scan_time; +  private int last_scan_time; +  +  //! Also deletes old files. +  //! +  // Will not scan for files if compressor is running. This means we might not +  // remove an old file because the compressor is running but that does not +  // matter since this function is ran so often. Sooner or later files will be +  // compressed (if there is a compressor) and old files will be deleted (if +  // days_to_keep_files > 0).    private void compress_logs(string fname, string active_log)    { -  if(!compressor_program || !sizeof(compressor_program)) -  return; // No compressor program specified... +  if(!compressor_exists && !auto_file_removal) +  // No compressor program specified, nor is auto file removal active... +  return;    if(compressor_process && !compressor_process->status()) -  return; // The compressor is already running... -  if(time(1) - last_compressor_scan_time < 300) -  return; // Scan for compressable files at most once every 5 minutes... -  last_compressor_scan_time = time(1); +  return; // The compressor is running... +  if(time(1) - last_scan_time < 300) +  return; // Scan for files at most once every 5 minutes... +  last_scan_time = time(1);    fname = roxen_path(fname);    active_log = roxen_path(active_log);    string dir = dirname(fname); -  +  int min_mtime = time(1) - (days_to_keep_files * 24 * 60 * 60); +  string pattern = "^"+replace(basename(fname), +  ({ "%y", "%m", "%d", "%h", "%H" }), +  ({ "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", +  "[0-9][0-9]", "[0-9][0-9]", "(.+)" })); +  Regexp regexp = Regexp(pattern); +  Regexp regexp_non_compressed = Regexp(pattern + "$");    foreach(sort(get_dir(dir) || ({})), string filename_candidate)    {    if(filename_candidate == basename(active_log)) -  +  {    continue; // Don't try to compress the active log just yet... -  if(Regexp("^"+replace(basename(fname), -  ({ "%y", "%m", "%d", "%h", "%H" }), -  ({ "[0-9][0-9][0-9][0-9]", "[0-9][0-9]", -  "[0-9][0-9]", "[0-9][0-9]", "(.+)" }))+"$")-> -  match(filename_candidate)) +  } +  else if(compressor_exists && +  regexp_non_compressed->match(filename_candidate))    {    string compress_file = combine_path(dir, filename_candidate);    Stdio.Stat stat = file_stat(compress_file);    if(!stat || time(1) < stat->mtime + 1200)    continue; // Wait at least 20 minutes before compressing log file...    werror("Compressing log file %O\n", compress_file);    compressor_process = Process.Process(({ compressor_program,    compress_file }));    return;    } -  +  else if(auto_file_removal && regexp->match(filename_candidate)) +  { +  // Wipe the file if it is old. +  string log_file = combine_path(dir, filename_candidate); +  Stdio.Stat stat = file_stat(log_file, 1); // 1 means symlinks will not be followed. +  if(stat->isreg && stat->mtime < min_mtime) +  { +  werror("Deleting log file %O due to old age.\n", log_file); +  rm(log_file);    }    } -  +  } +  }       private void do_open_co() { handle(do_open); }    private void do_open(void|object mutex_key)    {    if (!mutex_key) mutex_key = lock->lock();       mixed parent;    if (catch { parent = function_object(object_program(this_object())); } ||    !parent) {    // Our parent (aka the configuration) has been destructed.