Roxen.git / server / protocols / http.pike

version» Context lines:

Roxen.git/server/protocols/http.pike:1:   // This is a roxen protocol module.   // Modified by Francesco Chemolli to add throttling capabilities.   // Copyright © 1996 - 2009, Roxen IS.    - constant cvs_version = "$Id: http.pike,v 1.643 2012/01/24 16:15:47 grubba Exp $"; + constant cvs_version = "$Id$";   // #define REQUEST_DEBUG   #define MAGIC_ERROR      #define REQUESTID this      #ifdef MAGIC_ERROR - inherit "highlight_pike"; + //inherit "highlight_pike";   #endif      // HTTP protocol module.   #include <roxen.h>   #define TIMER_PREFIX "http:"   #include <timers.h>      inherit RequestID;      #ifdef PROFILE   #define HRTIME() gethrtime()   int req_time = HRTIME();   #endif      #ifdef REQUEST_DEBUG   int footime, bartime; - #define REQUEST_WERR(X) do {bartime = gethrtime()-footime; werror("%s (%d)\n", (X), bartime);footime=gethrtime();} while (0) + #define REQUEST_WERR(X) do { \ +  if (this) { \ +  bartime = gethrtime()-footime; \ +  werror("%s (%d)\n", (X), bartime); \ +  footime=gethrtime(); \ +  } else { \ +  werror("%s\n", (X)); \ +  } \ +  } while (0)   #else   #define REQUEST_WERR(X) do {} while (0)   #endif      #ifdef FD_DEBUG   #ifdef REQUEST_DEBUG   #define FD_WERR(X) REQUEST_WERR(X)   #else   #define FD_WERR(X) werror("%s\n", (X))   #endif
Roxen.git/server/protocols/http.pike:319: Inside #if defined(OLD_RXML_CONFIG)
   if(sscanf(replace(raw_url,({"%3c","%3e","%3C","%3E" }),    ({"<",">","<",">"})),"/<%*s>/%s",url)!=2)    url = "/";    else    url = "/"+url;       multiset do_mod_config( multiset config )    {    if(!mod_config) return config;    foreach(mod_config, string m) -  if(m[0]=='-') +  if(sizeof(m) && m[0]=='-')    config[m[1..]]=0;    else    config[m]=1;    return config;    };       void do_send_reply( string what, string url ) {    // FIXME: Delayed chaining! my_fd_busy.    CHECK_FD_SAFE_USE;    url = url_base() + url[1..];
Roxen.git/server/protocols/http.pike:394: Inside #if 0
   array tmp = arrayp(contents) ? contents : ({ contents});       foreach(tmp, string cookieheader) {       foreach(((cookieheader/";") - ({""})), string c)    {    string name, value;    while(sizeof(c) && c[0]==' ') c=c[1..];    if(sscanf(c, "%s=%s", name, value) == 2)    { +  // NB: Assume invalidly %-encoded values are unencoded. +  // Fixes [bug 7818]. +  catch {    value=http_decode_string(value); -  +  }; +  catch {    name=http_decode_string(name); -  +  };    cookies[ name ]=value;   #ifdef OLD_RXML_CONFIG    if( (name == "RoxenConfig") && strlen(value) )    config = mkmultiset( value/"," );   #endif    }    }    }    return cookies;   }
Roxen.git/server/protocols/http.pike:481:    REQUEST_WERR(sprintf("Repaired query=%O\n", query));    }    }       {    mapping(string:array(string)) vars =    misc->post_variables ? misc->post_variables + ([]) : ([]);    if (query)    split_query_vars (query, vars);    +  if (mixed err = catch { +  f = http_decode_string(f); +  }) { +  report_debug("Client %O sent an invalidly %%-encoded query: %O: %s", +  client_var->fullname, f, describe_error(err)); +  return invalid_request(); +  } +     if (sizeof (vars)) {    decode_query: {    if (input_charset) {    if (mixed err = catch { -  f = decode_query_charset (_Roxen.http_decode_string (f), -  vars, input_charset); +  f = decode_query_charset(f, vars, input_charset);    }) {    report_debug ("Client %O sent query %O which failed to decode "    "with its own charset %O: %s",    client_var->fullname, raw_url,    input_charset, describe_error (err));    input_charset = 0;    }    else break decode_query;    }    -  f = decode_query_charset (_Roxen.http_decode_string (f), -  vars, "roxen-http-default"); +  f = decode_query_charset(f, vars, "roxen-http-default");    }       if (array(string) rest = m_delete (real_variables, ""))    rest_query = rest[0];       if (input_charset && misc->post_variables) {    if (query) {    // Complicated to repopulate misc->post_variables with    // decoded values.    function(string:string) decoder =    Roxen.get_decoder_for_client_charset (input_charset);    mapping(string:array(string)) decoded_vars = ([]);    foreach (misc->post_variables; string var; array(string) vals) { -  var = decoder (var); +  string dec_var = decoder (var);    // We know the post_variables are first in the array values. -  decoded_vars[var] = real_variables[var][..sizeof (vals) - 1]; +  if (real_variables[dec_var]) { +  decoded_vars[dec_var] = +  real_variables[dec_var][..sizeof (vals) - 1]; +  } else { +  report_debug("Lost track of posted variable %O(%O).\n" +  "Original value: %O\n", +  dec_var, var, vals);    } -  +  }    misc->post_variables = decoded_vars;    }    else    misc->post_variables = real_variables + ([]);       // Ensure that the variable names in misc->post_parts and    // misc->files are decoded too.    if (misc->post_parts) {    mapping(string:string) decoded_names = ([]);    function(string:string) decoder =
Roxen.git/server/protocols/http.pike:546:    if (misc->files)    misc->files = map (misc->files, decoded_names);    }    }    }       else {    // No variables to decode, only the path. Optimize    // decode_query with "roxen-http-default" since we know    // there's no magic_roxen_automatic_charset_variable. -  f = http_decode_string (f); +     if (input_charset) {    if (mixed err = catch {    f = Roxen.get_decoder_for_client_charset (input_charset) (f);    })    report_debug ("Client %O requested path %O which failed to decode "    "with the input charset %O: %s",    client_var->fullname, f, input_charset,    describe_error (err));    }    else    catch (f = utf8_to_string (f)); -  +  if (String.width(f) > 8) { +  // Wide, so it might contain combiners. +  // Combine them if they are there. +  f = Unicode.normalize(f, "NFC");    } -  +  }       // Now after charset decode we can add the multipart/form-data    // variables in case they were rfc 2388 correct.    if (mapping(string:array(string)) ppv =    m_delete (misc, "proper_post_vars")) {    misc->post_variables = ppv;    foreach (ppv; string var; array(string) val) {    if (array(string) oldval = real_variables[var])    // Ensure POST variables come first, for consistency.    real_variables[var] = val + oldval;
Roxen.git/server/protocols/http.pike:612:    }    }    }       if (has_prefix (f, "/"))    not_query = combine_path_unix ("/", f);    else    not_query = combine_path_unix ("/", f)[1..];    }    -  { -  int i = search (client, "MSIE"); -  if (i < 0) +     supports->vary = 1; -  else if (++i < sizeof (client) && -  sscanf (client[i], "%d", int msie_major) == 1) { -  // Vary doesn't work in MSIE <= 6. -  // FIXME: Does it really work in MSIE 7? -  if (msie_major >= 7) -  supports->vary = 1; -  } -  } +        //REQUEST_WERR("HTTP: parse_got(): supports");    if(!referer) referer = ({ });       if(misc->proxyauth)    {    // The Proxy-authorization header should be removed... So there.    mixed tmp1,tmp2;       foreach(tmp2 = (raw / "\n"), tmp1) {
Roxen.git/server/protocols/http.pike:867:    if( !res )    {    TIMER_END(parse_got);    return 0; // Not enough data    }    [data, line, request_headers] = res;    }    hp = 0;    TIMER_END(parse_got);    +  do { +  mapping tmp = ([ +  "event" : "BEGIN_REQUEST", +  "tid" : this_thread()->id_number(), +  "remote" : remoteaddr, +  ]); +  +  // Indicate parent request if one exists +  if (objectp(root_id) && (root_id != root_id->root_id)) { +  tmp->subrequest = Val.True; // We should get rid of this one +  tmp->prid = root_id->request_uuid; +  werror("Subrequest detected!\n"); +  } +  json_logger->log(tmp); +  } while(0); +  +     // The following was earlier a separate function parse_got_2.       TIMER_START(parse_got_2);    TIMER_START(parse_got_2_parse_line);    string f;    array(string) sl = line / " ";    switch( sizeof( sl ) )    {    default:    sl = ({ sl[0], sl[1..sizeof(sl)-2]*" ", sl[-1] });
Roxen.git/server/protocols/http.pike:912:       case 2: // HTTP/0.9    case 1: // PING    misc->connection = "close";    method = sl[0];    f = sl[-1];    if( sizeof( sl ) == 1 )    sscanf( method, "%s%*[\r\n]", method );       clientprot = prot = "HTTP/0.9"; -  if(method != "PING") -  method = "GET"; // 0.9 only supports get. -  else -  { +  if(method == "PING") {    // FIXME: my_fd_busy.   #ifdef CONNECTION_DEBUG    werror ("HTTP[%s]: Response =============================================\n"    "%O\n", DEBUG_GET_FD, "PONG\r\n");   #else    REQUEST_WERR("HTTP: Send PONG");   #endif    my_fd->write("PONG\r\n");    TIMER_END(parse_got_2_parse_line);    TIMER_END(parse_got_2);    return 2; -  } +  } else if (method != "CONNECT") +  method = "GET"; // 0.9 only supports get. +     data = ""; // no headers or extra data...    sscanf( f, "%s%*[\r\n]", f );    if (sizeof(sl) == 1)    NO_PROTO_CACHE();    break;       case 0:    /* Not reached */    break;    }
Roxen.git/server/protocols/http.pike:961:    sscanf(remoteaddr, "%s %*s", remoteaddr);    }    if(!remoteaddr) {    REQUEST_WERR(sprintf ("HTTP: No remote address (error %d).",    my_fd->errno()));    TIMER_END(parse_got_2);    return 2;    }    }    +  json_logger->log(([ +  "event" : "GOT_HEADERS", +  "headers" : request_headers, +  "prot" : prot, +  "data_length" : strlen(data), +  ])); +     TIMER_START(parse_got_2_parse_headers); -  +  array(int|string) new_forwarded_header = ({});    foreach (request_headers; string linename; array|string contents)    { -  if( arrayp(contents) ) contents = contents[0]; +  array(string) acontents; +  if( arrayp(contents) ) { +  acontents = contents; +  contents = contents[0]; +  } else { +  acontents = ({ contents }); +  }    switch (linename)    {    case "cache-control": // Opera sends "no-cache" here.    case "pragma": pragma|=(multiset)((contents-" ")/","); break;       case "content-length": misc->len = (int)contents; break;    case "transfer-encoding":    misc->transfer_encoding =    map(lower_case(contents)/";", String.trim_all_whites);    if (has_value(misc->transfer_encoding, "chunked")) {
Roxen.git/server/protocols/http.pike:1063:    if (mixed err = catch {    contents = http_decode_string (Standards.URI(contents,    "dummy:")->path);    }) {   #ifdef DEBUG    report_debug(sprintf("Destination header contained a bad URI: %O\n"    "%s", contents, describe_error(err)));   #endif /* DEBUG */    }    misc["new-uri"] = VFS.normalize_path (contents); +  if (String.width(misc["new-uri"]) > 8) { +  misc["new-uri"] = Unicode.normalize(misc["new-uri"], "NFC"); +  }    break;       case "expect":    // RFC 2616, section 14.20: "A server that does not    // understand or is unable to comply with any of the    // expectation values in the Expect field of a request MUST    // respond with appropriate error status." We only handle the    // standard 100-continue case (see ready_to_receive).    // Also: "Comparison of expectation values is case-insensitive    // for unquoted tokens (including the 100-continue token), and
Roxen.git/server/protocols/http.pike:1086: Inside #if defined(CONNECTION_DEBUG)
  #ifdef CONNECTION_DEBUG    werror ("HTTP[%s]: Response (length %d) =================================\n"    "%O\n", DEBUG_GET_FD, sizeof (res), res);   #else    REQUEST_WERR(sprintf("HTTP: Send %O", res));   #endif    my_fd->write (res);    return 2;    }    break; +  +  case "forwarded": +  // RFC 7239 +  misc->forwarded = map(map(acontents, MIME.tokenize), `/, ({ ',' })) * ({}); +  break; +  +  // The X-Forwarded-* headers have been obsoleted by RFC 7239. +  // Convert them to the corresponding forwarded header as per +  // RFC 7239 7.4. But only if it hasn't already been done. +  case "x-forwarded-for": +  if (request_headers->forwarded) break; +  foreach(acontents, string line) { +  foreach(MIME.tokenize(line) / ({ ',' }), array(string|int) segment) { +  string ip; +  if ((sizeof(segment) == 1) && stringp(segment[0])) { +  ip = segment[0]; +  } else { +  ip = MIME.quote(segment);    } -  +  if (has_value(ip, ":") && !has_value(ip, ".") && +  !has_prefix(ip, "[")) { +  // DWIM IPv6 +  ip = "[" + ip + "]";    } -  +  if (sizeof(new_forwarded_header)) { +  new_forwarded_header += ({ ',' }); +  } +  new_forwarded_header += ({ "for", '=', ip }); +  } +  } +  break; +  case "x-forwarded-by": +  case "x-forwarded-host": +  case "x-forwarded-proto": +  if (request_headers->forwarded) break; +  string field = linename[sizeof("x-forwarded-")..]; +  foreach(acontents, string line) { +  foreach(MIME.tokenize(line) / ({ ',' }), array(string|int) segment) { +  string value; +  if ((sizeof(segment) == 1) && stringp(segment[0])) { +  value = segment[0]; +  } else { +  value = MIME.quote(segment); +  } +  if (sizeof(new_forwarded_header)) { +  new_forwarded_header += ({ ',' }); +  } +  new_forwarded_header += ({ field, '=', value }); +  } +  } +  break; +  } +  } +  if (sizeof(new_forwarded_header)) { +  request_headers->forwarded = MIME.quote(new_forwarded_header); +  misc->forwarded = new_forwarded_header / ({ ',' }); +  }    TIMER_END(parse_got_2_parse_headers);    }       TIMER_START(parse_got_2_more_data);    if (misc->chunked) {    int ret = got_chunk_fragment(new_data);    if (ret != 3) {    REQUEST_WERR(sprintf("HTTP: More data needed in %s (chunked).", method));    ready_to_receive();    TIMER_END(parse_got_2_more_data);
Roxen.git/server/protocols/http.pike:1235:       foreach (parts, MIME.Message part) {    string name = part->disp_params->name;    if (string decoded = rfc2047_decoded[name]) name = decoded;       post_parts[name] += ({part});       string data = part->getdata();    if (string charset = part->param->charset) {    if (mixed err = catch { -  data = Locale.Charset.decoder (charset)-> +  data = Charset.decoder (charset)->    feed (data)->drain();    })    report_debug ("Client %q sent data for %q which failed to "    "decode with supplied charset %q: %s",    client_var->fullname, name, charset,    describe_error (err));    }    post_vars[name] += ({data});       if (part->headers["content-type"])
Roxen.git/server/protocols/http.pike:1343:       MERGE_TIMERS(conf);    destruct();   }      protected void cleanup_request_object()   {    if( conf )    conf->connection_drop( this_object() );    xml_data = 0; +  request_uuid = UNDEFINED;   }      void end(int|void keepit)   {    if (my_fd) {    CHECK_FD_SAFE_USE;    }       cleanup_request_object();   
Roxen.git/server/protocols/http.pike:1380:       o->host = host;    o->conf = conf;    o->my_fd_busy = !!pipe;    o->pipe = 0;    o->connection_misc = connection_misc;    o->kept_alive = kept_alive+1;    object fd = my_fd;    my_fd=0;    pipe = 0; -  chained_to = o; +  chained_to = o->my_fd_released;    call_out (o->chain, 0, fd,port_obj,leftovers);    disconnect();    return;    }       data_buffer = 0;    pipe = 0;    disconnect();   }   
Roxen.git/server/protocols/http.pike:1439:    my_fd = 0;       disconnect();   }      protected void do_timeout()   {    int elapsed = predef::time(1)-time;    if(time && elapsed >= 30)    { +  // FIXME: Hardcoded timeout of 30 seconds above.   #ifdef CONNECTION_DEBUG    werror ("HTTP[%s]: Connection timed out. Closing.\n", DEBUG_GET_FD);   #endif    REQUEST_WERR(sprintf("HTTP: Connection timed out. Closing.\n"    "rcb:%O\n"    "wcb:%O\n"    "ccb:%O\n",    my_fd->query_read_callback(),    my_fd->query_write_callback(),    my_fd->query_close_callback()));    MARK_FD("HTTP timeout"); -  +  if (my_fd && my_fd->linger) { +  // We don't care if there's still data in the buffers +  // (there probably is), just terminate the connection +  // as soon as possible. +  my_fd->linger(0); +  }    end();    } else {   #ifdef DEBUG    error ("This shouldn't happen.\n");   #endif    // premature call_out... *¤#!"    call_out(do_timeout, 10);    MARK_FD("HTTP premature timeout");    }   }
Roxen.git/server/protocols/http.pike:1527:    <tr>    <td colspan='2'>" +    (body ? #"    <div class='msg'>" + body + #"</div>" : "") + #"    </td>    </tr>    <tr valign='bottom' height='100%'>    <td colspan='2'>    <table border='0' cellspacing='0' cellpadding='0'>    <tr> -  <td><img src='/internal-roxen-roxen-mini.gif' /></td> +  <td><img src='/internal-roxen-roxen-mini' /></td>    <td class='info'>    &nbsp;&nbsp;<b>" + roxen_product_name + #"</b>    <font color='#ffbe00'>|</font> " + roxen_dist_version + #"    </td>    </tr>    </table>    </td>    </tr>   </table>   </body></html>";
Roxen.git/server/protocols/http.pike:1690:    function dcl = d->describe_comma_list;    bt = ({});       foreach (reverse (err[1]), mixed ent) {    string file, func, descr;    int line;    if (arrayp (ent)) {    if (sizeof (ent) && stringp (ent[0]))    if (has_prefix (ent[0], cwd))    file = ent[0] = ent[0][sizeof (cwd)..]; -  else if (has_prefix (ent[0], roxenloader.server_dir)) -  file = ent[0] = ent[0][sizeof (roxenloader.server_dir)..]; +  else if (has_prefix (ent[0], roxenloader.server_dir + "/")) +  file = ent[0] = ent[0][sizeof (roxenloader.server_dir + "/")..];    else    file = ent[0];    if (sizeof (ent) >= 2) line = ent[1];    if (sizeof (ent) >= 3)    if(functionp(ent[2])) {    func = "";    if (object o = function_object (ent[2])) {    string s;    if (!catch (s = sprintf ("%O",o)) && s != "object")    func = s + "->";
Roxen.git/server/protocols/http.pike:1741:    mapping e = roxen.query_var("errors");    if(e) {    array r = e[(int)eid];    if (r && (md5 == String.string2hex(Crypto.MD5.hash(r[3])))) {    return r;    }    }    return 0;   }    + void send_internal_error (array err) + { +  internal_error (err); +  send_result(); + }      void internal_error(array _err)   {    NO_PROTO_CACHE();    mixed err = _err;    _err = 0; // hide in backtrace, they are bad enough anyway...    array err2;    if(port_obj && port_obj->query("show_internals"))    {    err2 = catch {
Roxen.git/server/protocols/http.pike:1775:    Roxen.http_low_answer(500, error_page("The server failed to fulfill "    "your query."));    }    report_error("Internal server error: " +    describe_backtrace(err) + "\n");   #ifdef INTERNAL_ERROR_DEBUG    report_error(sprintf("Raw backtrace:%O\n", err));   #endif /* INTERNAL_ERROR_DEBUG */   }    + protected int(1..1) invalid_request() + { +  file = Roxen.http_low_answer(400, "<html><body><h1>Client error: Invalid request.</h1></body></html>\n"); +  roxen.handle(send_result); +  return 1; + } +    // This macro ensures that something gets reported even when the very   // call to internal_error() fails. That happens eg when this_object()   // has been destructed.   #define INTERNAL_ERROR(err) do { \    if (mixed __eRr = catch (internal_error (err))) \    report_error("Internal server error: " + describe_backtrace(err) + \    "internal_error() also failed: " + describe_backtrace(__eRr)); \    } while (0)      int wants_more()   {    return !!cache;   }    - protected object(this_program) chained_to; + protected function(:void) chained_to;      protected void destroy()   {    // To avoid references to destructed RequestID objects. Happens    // otherwise when prot_https makes a http -> https redirect, for    // instance.    remove_call_out (do_timeout);       if (chained_to) {    // This happens when do_log() is called before the request    // has been chained (eg for short data over fast connections). -  call_out(chained_to->my_fd_released, 0); +  call_out(chained_to, 0);    chained_to = 0;    }   }      void do_log( int|void fsent )   {   #ifdef CONNECTION_DEBUG    werror ("HTTP[%s]: Response sent ========================================\n",    DEBUG_GET_FD);   #endif    MARK_FD("HTTP logging"); // fd can be closed here    if (chained_to) {    // Release the other sender. -  call_out(chained_to->my_fd_released, 0); +  call_out(chained_to, 0);    chained_to = 0;    }    TIMER_START(do_log);    if(conf)    {    if(!(file->len = fsent) && pipe) {    file->len = pipe->bytes_sent();    }    if(conf)    {
Roxen.git/server/protocols/http.pike:1864:    if (t > 0.05) conf->queue_num_runs_005s++;    if (t > 0.15) conf->queue_num_runs_015s++;    if (t > 0.50) conf->queue_num_runs_05s++;    if (t > 1.00) conf->queue_num_runs_1s++;    if (t > 5.00) conf->queue_num_runs_5s++;    if (t > 15.00) conf->queue_num_runs_15s++;    conf->queue_acc_time += (int)(1E6*t);    }       conf->log(file, this_object()); +  +  json_logger->log(([ +  "event" : "LOG_REQUEST", +  "htime" : handle_time, +  "qtime" : queue_time, +  "rtime" : gethrtime()-hrtime, +  ]));    }    }    if( !port_obj )    {    TIMER_END(do_log);    MERGE_TIMERS(conf);    if( conf )    conf->connection_drop( this_object() );    call_out (disconnect, 0);    return;
Roxen.git/server/protocols/http.pike:2511: Inside #if defined(HTTP_COMPRESSION)
   return 0;   }      #endif // HTTP_COMPRESSION      // Send the result.   void send_result(mapping|void result)   {    TIMER_START(send_result);    +  json_logger->log(([ +  "event" : "SEND_RESULT_BEGIN", +  ])); +     if (my_fd)    CHECK_FD_SAFE_USE;       destruct_threadbound_session_objects();       handle_time = gethrtime() - handle_time;   #if constant(System.CPU_TIME_IS_THREAD_LOCAL)    handle_vtime = gethrvtime() - handle_vtime;   #endif   
Roxen.git/server/protocols/http.pike:2569:    Roxen.http_status_messages[misc->error_code]);    else if(err = catch {    file = conf->error_file( this_object() );    })    INTERNAL_ERROR(err);    }    else    {    if((file->file == -1) || file->leave_me)    { +  json_logger->log(([ +  "event" : "SEND_RESULT_END", +  "msg" : "No result", +  ]));    TIMER_END(send_result);    file = 0;    pipe = 0;    if (do_not_disconnect) return;    my_fd = 0;    return;    }       if(file->type == "raw") file->raw = 1;    }
Roxen.git/server/protocols/http.pike:2614:    file->data = "";    }       string head_status = file->rettext;    if (head_status) {    if (!file->file && !file->data &&    (!file->type || file->type == "text/html")) {    // If we got no body then put the message there to make it    // more visible.    file->data = "<html><body>" + -  replace (Roxen.html_encode_string (head_status), "\n", "<br />\n") + +  replace (Roxen.html_encode_string (string_to_utf8(head_status)), +  "\n", "<br />\n") +    "</body></html>";    file->len = sizeof (file->data); -  file->type = "text/html"; +  file->type = "text/html; charset=utf-8";    }    if (has_value (head_status, "\n"))    // Fold lines nicely.    head_status = map (head_status / "\n", String.trim_all_whites) * " ";    }      #ifdef HTTP_COMPRESSION    if(compression_enabled_for_mimetype (get_response_content_type (file))) {    // Notify proxies etc. that we depend on the Accept-Encoding header,    // but we don't want to register it as a vary callback since it's    // handled directly by the protocol cache.    if(!misc->vary)    misc->vary = (< "accept-encoding" >);    else    misc->vary["accept-encoding"] = 1;    }   #endif       mapping(string:string) heads = make_response_headers (file);    -  +  if (heads["Set-Cookie"]) { +  // Never protocol cache setting of cookies. +  NO_PROTO_CACHE(); +  } +     // Notes about the variant headers:    //    // Date Changes with every request.    // Content-Type May change if a byte-range request is performed.    // Content-Length May change due to If-* headers, etc.    // Connection Depends on the protocol version and state.    // Expires Might need to modify this for HTTP/1.0 clients.    // Cache-Control Might need to modify this when sending stale responses.    mapping(string:string) variant_heads = ([ "Date":"",    "Content-Type":"",
Roxen.git/server/protocols/http.pike:2710:    }       int varies = misc->vary && (sizeof(misc->vary) - misc->vary["host"]);   #ifdef RAM_CACHE    int stored_in_cache;    if( (misc->cacheable > 0) && !misc->no_proto_cache)    {    if ((<"HEAD","GET">)[method]) {    if( file->len>0 && // known length.    ((file->len + sizeof(head_string)) < -  conf->datacache->max_file_size) +  conf->datacache->get_cache_stats()->max_file_size)    // vvv Relying on the interpreter lock from here.    && misc->cachekey )    {    misc->cachekey->activate();    // ^^^ Relying on the interpreter lock to here.    string data = "";    if( file->data ) data = file->data[..file->len-1];    if( file->file ) data = file->file->read(file->len);      #ifdef HTTP_COMPRESSION
Roxen.git/server/protocols/http.pike:2931:    string if_range;    if (if_range = request_headers["if-range"]) {    // Check If-Range header (RFC 2068 14.27).    if (has_prefix(if_range, "\"")) {    // ETag    if (if_range != misc->etag) {    // ETag has changed.    skip = 1;    }    } else { +  // Needs strict equality according to RFC 7233    array(int) since_info = Roxen.parse_since(if_range); -  if (!since_info || (since_info[0] < misc->last_modified)) { +  if (!since_info || (since_info[0] != misc->last_modified)) {    // Failed to parse since info, or the file has changed.    skip = 1;    }    }    }    if (!skip) {    // NOTE: Modifies both arguments destructively.    handle_byte_ranges(file, variant_heads);    }    }
Roxen.git/server/protocols/http.pike:3006:    report_error("Internal server error: " + describe_backtrace(err));    }   }      void handle_request()   {    if (mixed err = catch {       REQUEST_WERR("HTTP: handle_request()");    TIMER_START(handle_request); +  json_logger->log(([ +  "event" : "HANDLE_REQUEST_BEGIN", +  "qtime" : queue_time, +  ])); + #ifdef TIMERS + #define LOG_HANDLE_END() do { json_logger->log((["event" : "HANDLE_REQUEST_END", "handle_time" : timers->handle_request ])); } while(0) + #else + #define LOG_HANDLE_END() + #endif    -  +    #ifdef MAGIC_ERROR    if(prestate->old_error)    {    array err = get_error(variables->error, variables->error_md5 || "NONE");    if(err && arrayp(err))    {    if(prestate->plain)    {    file = ([    "type":"text/plain",    "data":generate_bugreport( @err ),    ]);    TIMER_END(handle_request); -  +  LOG_HANDLE_END()    send_result();    return;    } else {    if(prestate->find_file)    {    if (!roxen.configuration_authenticate (this_object(), "View Settings"))    file = Roxen.http_auth_required("admin");    else    file = ([    "type":"text/html",    "data":handle_error_file_request( @err ),    ]);    TIMER_END(handle_request); -  +  LOG_HANDLE_END();    send_result();    return;    }    }    }    }   #endif /* MAGIC_ERROR */       MARK_FD("HTTP handling request");   
Roxen.git/server/protocols/http.pike:3068:    if (this)    destruct_threadbound_session_objects();       if(e)    INTERNAL_ERROR( e );       else {    if (result && result->pipe) {    REQUEST_WERR("HTTP: handle_request: pipe in progress.");    TIMER_END(handle_request); +  json_logger->log(([ +  "event" : "PIPE_BEGIN", +  ])); +  LOG_HANDLE_END();    return;    }    file = result;    }       if( file && file->try_again_later )    {    if( objectp( file->try_again_later ) )    ;    else    call_out( roxen.handle, file->try_again_later, handle_request );    TIMER_END(handle_request);    return;    }       TIMER_END(handle_request);    send_result(); -  +  LOG_HANDLE_END();    }) {    call_out (disconnect, 0);    report_error("Internal server error: " + describe_backtrace(err));    }   }      /* We got some data on a socket.    * =================================================    */   // array ccd = ({});
Roxen.git/server/protocols/http.pike:3235:    // To allow for transition to absoluteURIs in all requests in future    // versions of HTTP, all HTTP/1.1 servers MUST accept the absoluteURI    // form in requests, even though HTTP/1.1 clients will only generate    // them in requests to proxies.    misc->prot_cache_key = raw_url;    if (has_prefix(raw_url, port_obj->url_prefix)) {    sscanf(raw_url[sizeof(port_obj->url_prefix)..], "%[^/]%s",    misc->host, raw_url);    }    -  string canon_hostport; +  string canon_host = "*:";    if (string host = misc->host) {    // Parse and canonicalize the host header for use in the url    // used for port matching.    int port = port_obj->default_port;    if (has_prefix(host, "[")) {    // IPv6 address    sscanf(lower_case(host), "[%s]:%d", host, port);    host = Protocols.IPv6.normalize_addr_basic (host) || host; -  canon_hostport = "[" + host + "]:" + port; +  canon_host = "[" + host + "]:";    } else {    sscanf(lower_case(host), "%[^:]:%d", host, port); -  canon_hostport = host + ":" + port; +  canon_host = host + ":";    }    misc->hostname = host;    misc->port = port;    }       if( !conf || !(path = port_obj->path ) ||    (sizeof( path ) && !has_prefix(raw_url, path)) ) { -  // FIXME: port_obj->name & port_obj->default_port are constant +  // FIXME: port_obj->name & port_obj->port are constant    // consider caching them?    -  +  // NB: Ignore the port number from the host header here, as +  // the client may access us via a load balancing proxy +  // or similar on a different port than our actual port. +  // cf [bug 7385].    string port_match_url = (port_obj->url_prefix + -  (canon_hostport || ("*:" + port_obj->port)) + +  canon_host + port_obj->port +    raw_url);    conf = port_obj->find_configuration_for_url(port_match_url, this);       // Note: The call above might have replaced port_obj from one    // bound to a specific interface to one bound to ANY.       if (misc->defaulted_conf > 1)    // Use the full url in the cache if a fallback configuration    // (with or without the default_server flag) was chosen.    misc->prot_cache_key = port_match_url;    }    else if( strlen(path) )    adjust_for_config_path( path );       TIMER_END(find_conf);    -  +  json_logger->log(([ +  "event" : "GOT_REQUEST", +  "data_length" : strlen(data), +  "path" : path, +  "raw_url" : raw_url, +  ])); +  +  if (!conf->inited) { +  // Load the configuration from a handler thread, and then resume. +  roxen.handle(lambda(Configuration conf) { +  REQUEST_WERR(sprintf("HTTP: Loading configuration %O...", +  conf)); +  conf->enable_all_modules(); +  call_out(got_configuration, 0, conf); +  }, conf); +  return; +  } +  got_configuration(conf); +  }) +  { +  report_error("Internal server error: " + describe_backtrace(err)); +  disconnect(); +  } + } +  + protected void got_configuration(Configuration conf) + { +  if (mixed err = catch { +  REQUEST_WERR(sprintf("HTTP: Got configuration %O", conf)); +     // The "http_request_init" provider hook allows modules to do    // things very early in the request path, before the request is    // put in the handler queue and before the protocol cache is    // queried.    //    // Use with great care; this is run in the backend thread, and    // some things in the id object are still not initialized.    foreach (conf->get_providers ("http_request_init"), RoxenModule mod)    if (mapping res = mod->http_request_init (this)) {    conf->received += raw_bytes - sizeof(leftovers);    conf->requests++;    // RequestID.make_response_headers assumes the supports multiset exists.    if (!supports) supports = (<>);    send_result (res);    return;    }    -  +  int allow_protocache_lookup = 0;    if (rawauth)    {    /* Need to authenticate with the configuration */    NO_PROTO_CACHE(); -  +  allow_protocache_lookup = 1;    array(string) y = rawauth / " ";    realauth = 0;    auth = 0;    if (sizeof(y) >= 2)    {    y[1] = MIME.decode_base64(y[1]);    realauth = y[1];    }    }   
Roxen.git/server/protocols/http.pike:3335:    conf->requests++;       CHECK_FD_SAFE_USE;    my_fd->set_close_callback(0);    my_fd->set_read_callback(0);       remove_call_out(do_timeout);   #ifdef RAM_CACHE    TIMER_START(cache_lookup);    array cv; -  if(misc->cacheable && !misc->no_proto_cache && +  if(misc->cacheable && (!misc->no_proto_cache || allow_protocache_lookup) &&    (cv = conf->datacache->get(misc->prot_cache_key, this)) )    {    MY_TRACE_ENTER(sprintf("Checking entry %O", misc->prot_cache_key));    if( !cv[1]->key ) {    MY_TRACE_LEAVE("Entry invalid due to zero key");    conf->datacache->expire_entry(misc->prot_cache_key, this);    }    else    {    int can_cache = 1;
Roxen.git/server/protocols/http.pike:3541: Inside #if defined(RAM_CACHE)
   string if_range;    if (if_range = request_headers["if-range"]) {    // Check If-Range header (RFC 2068 14.27).    if (has_prefix(if_range, "\"")) {    // ETag    if (if_range != file->etag) {    // ETag has changed.    skip = 1;    }    } else { +  // Needs strict equality according to RFC 7233    array(int) since_info = Roxen.parse_since(if_range); -  if (!since_info || (since_info[0] < file->last_modified)) { +  if (!since_info || (since_info[0] != file->last_modified)) {    // Failed to parse since info, or the file has changed.    skip = 1;    }    }    }    if (!skip) {    file->data = d;    file->len = len;       // NOTE: Modifies both arguments destructively.
Roxen.git/server/protocols/http.pike:3626: Inside #if defined(RAM_CACHE) and #if undefined(RAM_CACHE_ASUME_STATIC_CONTENT)
   sprintf ("Entry out of date (disk: %s, cache: mtime %d)",    st ? "mtime " + st->mtime : "gone", file->mtime));   #endif    }    file = 0;    }    }    TIMER_END(cache_lookup);   #endif // RAM_CACHE    TIMER_START(parse_request); +  if (mixed err = catch {    if( things_to_do_when_not_sending_from_cache( ) )    return; -  +  }) { +  internal_error(err); +  roxen.handle(send_result); +  return; +  }    REQUEST_WERR(sprintf("HTTP: cooked headers %O", request_headers));    REQUEST_WERR(sprintf("HTTP: cooked variables %O", real_variables));    REQUEST_WERR(sprintf("HTTP: cooked cookies %O", cookies));    TIMER_END(parse_request);       REQUEST_WERR("HTTP: Calling roxen.handle().");    queue_time = gethrtime();    queue_length = roxen.handle_queue_length(); -  +  json_logger->log(([ +  "event" : "ENTER_QUEUE", +  "qlen" : queue_length, +  "vars" : real_variables, +  "cookies" : cookies, +  ]));    roxen.handle(handle_request_from_queue);    })    {    report_error("Internal server error: " + describe_backtrace(err));    disconnect();    }   }      /* Get a somewhat identical copy of this object, used when doing    * 'simulated' requests. */
Roxen.git/server/protocols/http.pike:3706:    c->raw_bytes = raw_bytes;    c->query = query;    c->not_query = not_query;    c->data = data;    c->extra_extension = extra_extension;       c->auth = auth;    c->realauth = realauth;    c->rawauth = rawauth;    c->since = since; +  +  c->json_logger = json_logger->child();    return c;   }      void clean()   {    if(!(my_fd && objectp(my_fd)))    end(); -  else if((predef::time(1) - time) > 4800) +  else if((predef::time(1) - time) > 4800) { +  // FIXME: Hardcoded timeout of 80 minutes above. +  if (my_fd->linger) my_fd->linger(0);    end();    } -  + }      protected void create(object f, object c, object cc)   {    if(f)    {   #if 0    if (f->query_accept_callback)    f->set_nonblocking(got_data, f->query_write_callback(), close_cb, 0, 0,    f->query_accept_callback());    else
Roxen.git/server/protocols/http.pike:3743:    time = predef::time(1);    hrtime = gethrtime();    call_out(do_timeout, 90);    }    root_id = this_object();   }      void chain(object f, object c, string le)   {    my_fd = f; -  +    #if defined(DEBUG) && defined(THREADS)    if (this_thread() != roxen->backend_thread)    error ("Not called from backend\n");   #endif       CHECK_FD_SAFE_USE;       port_obj = c;    MARK_FD("HTTP kept alive");    time = predef::time();