51354e2016-05-13Pontus Östlund //! A HTTP client that does async calls but waits for the result before //! returning the result. If the request takes too long the client will abort. //! The timeout can be set per request but is @tt{60@} seconds per default //! //! @example //! @code //! HTTPClient.Arguments args = HTTPClient.Arguments(([ //! "variables" : ([ "a" : 1 ]), // optional //! "maxtime" : 30 // optional //! ])); //! //! HTTPClient.Result res = HTTPClient.sync_get("http://the.url/to/fetch", args); //! //! if (res->ok) { //! werror("Call to %s was ok: %s\n", res->url, res->data[0..50]); //! } //! else { //! werror("Call to %s failed with status %d %s\n", //! res->url, res->status, res->status_description); //! } //! @endcode //! //! You can also do completly async requests with callbacks //! //! @example //! @code //! HTTPClient.Arguments args = HTTPClient.Arguments(); //! args->on_success(lambda (HTTPClient.Result resp) { //! werror("Successful request for %O\n", resp->url); //! }); //! //! args->on_failure(lambda (HTTPClient.Result resp) { //! werror("Failed request for %O (%d %s)\n", resp->url, //! resp->status, resp->status_description); //! }); //! //! HTTPClient.async_get("http://domain.com", args); //! @endcode
2612f72017-03-01Anders Johansson // #define HTTP_CLIENT_DEBUG
51354e2016-05-13Pontus Östlund  #ifdef HTTP_CLIENT_DEBUG # define TRACE(X...)werror("%s:%d: %s",basename(__FILE__),__LINE__,sprintf(X)) #else # define TRACE(X...)0 #endif protected constant DEFAULT_MAXTIME = 60; //! Do a HTTP GET request
8dfff62018-10-15Karl Gustav Sterneberg public Result sync_get(Protocols.HTTP.Session.URL uri, void|Arguments args, void|Session session)
51354e2016-05-13Pontus Östlund {
8dfff62018-10-15Karl Gustav Sterneberg  return do_safe_method("GET", uri, args, false, session);
51354e2016-05-13Pontus Östlund } //! Do a HTTP POST request
8dfff62018-10-15Karl Gustav Sterneberg public Result sync_post(Protocols.HTTP.Session.URL uri, void|Arguments args, void|Session session)
51354e2016-05-13Pontus Östlund {
8dfff62018-10-15Karl Gustav Sterneberg  return do_safe_method("POST", uri, args, false, session);
51354e2016-05-13Pontus Östlund } //! Do a HTTP PUT request
8dfff62018-10-15Karl Gustav Sterneberg public Result sync_put(Protocols.HTTP.Session.URL uri, void|Arguments args, void|Session session)
51354e2016-05-13Pontus Östlund {
8dfff62018-10-15Karl Gustav Sterneberg  return do_safe_method("PUT", uri, args, false, session);
51354e2016-05-13Pontus Östlund } //! Do a HTTP DELETE request
8dfff62018-10-15Karl Gustav Sterneberg public Result sync_delete(Protocols.HTTP.Session.URL uri, void|Arguments args, void|Session session)
51354e2016-05-13Pontus Östlund {
8dfff62018-10-15Karl Gustav Sterneberg  return do_safe_method("DELETE", uri, args, false, session);
51354e2016-05-13Pontus Östlund } //! Do an async HTTP GET request
8dfff62018-10-15Karl Gustav Sterneberg public void async_get(Protocols.HTTP.Session.URL uri, void|Arguments args, void|Session session)
51354e2016-05-13Pontus Östlund {
8dfff62018-10-15Karl Gustav Sterneberg  do_safe_method("GET", uri, args, true, session);
51354e2016-05-13Pontus Östlund } //! Do an async HTTP POST request
8dfff62018-10-15Karl Gustav Sterneberg public void async_post(Protocols.HTTP.Session.URL uri, void|Arguments args, void|Session session)
51354e2016-05-13Pontus Östlund {
8dfff62018-10-15Karl Gustav Sterneberg  do_safe_method("POST", uri, args, true, session);
51354e2016-05-13Pontus Östlund } //! Do an async HTTP PUT request
8dfff62018-10-15Karl Gustav Sterneberg public void async_put(Protocols.HTTP.Session.URL uri, void|Arguments args, void|Session session)
51354e2016-05-13Pontus Östlund {
8dfff62018-10-15Karl Gustav Sterneberg  do_safe_method("PUT", uri, args, true, session);
51354e2016-05-13Pontus Östlund } //! Do an async HTTP DELETE request
8dfff62018-10-15Karl Gustav Sterneberg public void async_delete(Protocols.HTTP.Session.URL uri, void|Arguments args, void|Session session)
51354e2016-05-13Pontus Östlund {
8dfff62018-10-15Karl Gustav Sterneberg  do_safe_method("DELETE", uri, args, true, session);
51354e2016-05-13Pontus Östlund } //! Fetch an URL with a timeout with the @[http_method] method. public Result do_safe_method(string http_method, Protocols.HTTP.Session.URL uri, void|Arguments args,
8dfff62018-10-15Karl Gustav Sterneberg  void|bool async, void|Session session)
51354e2016-05-13Pontus Östlund {
ea3bea2016-05-13Pontus Östlund  if (!args) { args = Arguments(); }
51354e2016-05-13Pontus Östlund  Result res;
8dfff62018-10-15Karl Gustav Sterneberg  Session s = session || Session();
298aef2017-04-25Karl Gustav Sterneberg  if (args && args->follow_redirects > -1) { s->follow_redirects = args->follow_redirects; }
51354e2016-05-13Pontus Östlund  object /* Session.Request */ qr; Thread.Queue q; mixed co_maxtime; if (args->maxtime) s->maxtime = args->maxtime; if (args->timeout) s->timeout = args->timeout; if (!async) { q = Thread.Queue(); }
1615cc2016-05-20Pontus Östlund  function(Result:void) cb = lambda (Result r) { TRACE("Got callback: %O\n", r); res = r; q && q->write("@"); if (async) { if (r->ok && args->on_success) { args->on_success(res); } else if (!r->ok && args->on_failure) { args->on_failure(r); } qr = 0; s = 0; } };
51354e2016-05-13Pontus Östlund  qr = s->async_do_method_url(http_method, uri, args->variables, args->data, args->headers,
1615cc2016-05-20Pontus Östlund  0, // headers received callback cb, // ok callback cb, // fail callback
aeb3892017-03-01Pontus Östlund  args->extra_args || ({}));
51354e2016-05-13Pontus Östlund  if (!query_has_maxtime()) {
aeb3892017-03-01Pontus Östlund  TRACE("No maxtime in Protocols.HTTP.Query. Set external max timeout: %O\n", s->maxtime || DEFAULT_MAXTIME);
51354e2016-05-13Pontus Östlund  co_maxtime = call_out(lambda () { TRACE("Timeout callback: %O\n", qr); res = Failure(([ "status" : 504, "status_desc" : "Gateway timeout", "host" : qr->con->host, "headers" : qr->con->headers, "url" : qr->url_requested ])); qr->set_callbacks(0, 0, 0, 0); qr->con->set_callbacks(0, 0, 0); destruct(qr);
8dfff62018-10-15Karl Gustav Sterneberg  s = 0;
51354e2016-05-13Pontus Östlund  q && q->write("@"); if (async) { if (args->on_failure) { args->on_failure(res); } qr = 0; s = 0; } }, s->maxtime || DEFAULT_MAXTIME); } if (!async) { q->read(); } if (co_maxtime && co_maxtime[0]) { TRACE("Remove timeout callout\n"); remove_call_out(co_maxtime); } if (!async) { q = 0; qr = 0; s = 0; } return res; } //! If the Protocols.HTTP.Query class doesn't have the maxtime property //! we need to take care of the maxtime timeout from the outside protected int(-1..1) _query_has_maxtime = -1; protected bool query_has_maxtime() { if (_query_has_maxtime != -1) { return _query_has_maxtime == 1; } Protocols.HTTP.Query q = Protocols.HTTP.Query(); _query_has_maxtime = (int) has_index(q, "maxtime"); destruct(q);
1615cc2016-05-20Pontus Östlund  return _query_has_maxtime == 1;
51354e2016-05-13Pontus Östlund } class Arguments { //! Data fetch timeout int timeout; //! Request timeout int maxtime;
298aef2017-04-25Karl Gustav Sterneberg  //! Number of times to follow redirects. A negative value will result in a //! default value being used. int follow_redirects = -1;
51354e2016-05-13Pontus Östlund  //! Additional request headers mapping(string:string) headers; //! Query variables mapping(string:mixed) variables; //! POST data
0b9ed42018-10-15Karl Gustav Sterneberg  void|string|mapping|Stdio.Stream data;
51354e2016-05-13Pontus Östlund  //! Callback to call on successful async request function(Result:void) on_success; //! Callback to call on failed async request function(Result:void) on_failure;
aeb3892017-03-01Pontus Östlund  //! Extra arguments that will end up in the @[Result] object array(mixed) extra_args;
51354e2016-05-13Pontus Östlund  //! If @[args] is given the indices that match any of this object's //! members will set those object members to the value of the
1615cc2016-05-20Pontus Östlund  //! corresponding mapping member.
51354e2016-05-13Pontus Östlund  protected void create(void|mapping(string:mixed) args) { if (args) {
aeb3892017-03-01Pontus Östlund  foreach (args; string key; mixed value) {
51354e2016-05-13Pontus Östlund  if (has_index(this, key)) {
aeb3892017-03-01Pontus Östlund  if ((< "variables", "headers" >)[key]) { // Cast all values to string value = mkmapping(indices(value), map(values(value), lambda (mixed s) { return (string) s; })); } this[key] = value; } else { error("Unknown argument %O!\n", key);
51354e2016-05-13Pontus Östlund  } } } } } //! HTTP result class. Consider internal. //! //! @seealso //! @[Success], @[Failure] class Result { //! Internal result mapping protected mapping result; //! Return @tt{true@} if scuccess, @tt{false@} otherwise public bool `ok(); //! The HTTP response headers public mapping `headers() { return result->headers; } //! The host that was called in the request public string `host() { return result->host; } //! The HTTP status of the response, e.g @tt{200@}, @tt{201@}, @tt{404@} //! and so on. public int `status() { return result->status; } //! The textual representation of @[status]. public string `status_description() { return result->status_desc; } //! Returns the requested URL public string `url() { return result->url; }
aeb3892017-03-01Pontus Östlund  //! Extra arguments set in the @[Arguments] object. public array(mixed) `extra_args() { return result->extra_args; }
51354e2016-05-13Pontus Östlund  //! @ignore protected void create(mapping _result) { //TRACE("this_program(%O)\n", _result->headers); result = _result; } //! @endignore } //! A class representing a successful request and its response. An instance of //! this class will be given as argument to the //! @[Concurrent.Future()->on_success()] callback registered on the returned //! @[Concurrent.Future] object from @[get_url()], @[post_url()], //! @[delete_url()], @[put_url()] or @[do_method()]. class Success { inherit Result; public bool `ok() { return true; } //! The response body, i.e the content of the requested URL
4b426d2017-04-12Pontus Östlund  public string `data() { string data = result->data; if (content_encoding && content_encoding == "gzip") { data = Gz.uncompress(data[10..<8], true); } return data; }
51354e2016-05-13Pontus Östlund  //! Returns the value of the @tt{content-length@} header. public int `length() { return headers && (int)headers["content-length"]; } //! Returns the content type of the requested document public string `content_type() { if (string ct = (headers && headers["content-type"])) { sscanf (ct, "%s;", ct); return ct; } }
4b426d2017-04-12Pontus Östlund  //! Returns the charset of the requested document, if given by the
51354e2016-05-13Pontus Östlund  //! response headers.
4b426d2017-04-12Pontus Östlund  public string `charset()
51354e2016-05-13Pontus Östlund  { if (string ce = (headers && headers["content-type"])) { if (sscanf (ce, "%*s;%*scharset=%s", ce) == 3) { if (ce[0] == '"' || ce[0] == '\'') { ce = ce[1..<1]; } return ce; } } }
4b426d2017-04-12Pontus Östlund  //! Returns the content encoding of the response if set by the remote server. public string `content_encoding() { if (string ce = (headers && headers["content-encoding"])) { return ce; } }
51354e2016-05-13Pontus Östlund } //! A class representing a failed request. An instance of //! this class will be given as argument to the //! @[Concurrent.Future()->on_failure()] callback registered on the returned //! @[Concurrent.Future] object from @[get_url()], @[post_url()], //! @[delete_url()], @[put_url()] or @[do_method()]. class Failure { inherit Result; public bool `ok() { return false; } }
8dfff62018-10-15Karl Gustav Sterneberg //! Class for the actual HTTP requests public class Session
51354e2016-05-13Pontus Östlund { inherit Protocols.HTTP.Session : parent; public int(0..) maxtime, timeout; public int maximum_connections_per_server = 20; Request async_do_method_url(string method, URL url, void|mapping query_variables,
0b9ed42018-10-15Karl Gustav Sterneberg  void|string|mapping|Stdio.Stream data,
51354e2016-05-13Pontus Östlund  void|mapping extra_headers, function callback_headers_ok, function callback_data_ok, function callback_fail, array callback_arguments) {
1615cc2016-05-20Pontus Östlund  if (stringp(url)) { url = Standards.URI(url); } // Due to a bug in Protocols.HTTP.Session which is fixed in Pike 8.1 // but not yet in 8.0. (2016-05-20) if (!extra_headers || !extra_headers->host || !extra_headers->Host) { extra_headers = extra_headers || ([]);
aeb3892017-03-01Pontus Östlund  TRACE("Set host in headers: %O\n", url); if (url->scheme == "http") { extra_headers->host = url->host; if (url->port != 80) { extra_headers->host += ":" + url->port; }
1615cc2016-05-20Pontus Östlund  }
aeb3892017-03-01Pontus Östlund  else if (url->scheme == "https") { extra_headers->host = url->host; if (url->port != 443) { extra_headers->host += ":" + url->port; }
1615cc2016-05-20Pontus Östlund  } if (!sizeof(extra_headers)) { extra_headers = 0; } TRACE("Host header set?: %O\n", extra_headers); }
4b426d2017-04-12Pontus Östlund  if (upper_case(method) == "POST") { TRACE("Got post request: %O, %O, %O, %O\n", url, query_variables, data, extra_headers); // In Pike < 8.1 the content-length header isn't auotmatically added so // we have to take care of that explicitly bool has_content_len = false; bool has_content_type = false; if (extra_headers) { array(string) lc_headers = map(indices(extra_headers), lower_case); if (has_value(lc_headers, "content-length")) { has_content_len = true; } if (has_value(lc_headers, "content-type")) { has_content_type = true; } } if (!has_content_len) { mapping(string:string) qvars = url->get_query_variables(); data = data||""; if (qvars && sizeof(qvars)) { if (!query_variables) { query_variables = qvars; } else { query_variables |= qvars; } } if (sizeof(data) && query_variables) { data += "&" + Protocols.HTTP.http_encode_query(query_variables); } else if (query_variables) { data = Protocols.HTTP.http_encode_query(query_variables); } if (!extra_headers) { extra_headers = ([]); } extra_headers["Content-Length"] = (string) sizeof(data); if (!has_content_type) { extra_headers["Content-Type"] = "application/x-www-form-urlencoded"; } query_variables = 0; } }
aeb3892017-03-01Pontus Östlund  TRACE("Request: %O\n", url);
0b9ed42018-10-15Karl Gustav Sterneberg  // NB: The argument data might be an instance of Stdio.Stream and // Protocols.HTTP.Session->async_do_method_url() does not declare that it // supports that. However, we know (and depend on) that // Protocols.HTTP.Session does not really care about the type. It just // passes the data-argument around. Eventually it ends up in an instance of // our class SessionQuery (see later in this file).
51354e2016-05-13Pontus Östlund  return ::async_do_method_url(method, url, query_variables, data, extra_headers, callback_headers_ok, callback_data_ok, callback_fail, callback_arguments); } class Request { inherit parent::Request;
aeb3892017-03-01Pontus Östlund  protected void set_extra_args_in_result(mapping(string:mixed) r) { if (extra_callback_arguments && sizeof(extra_callback_arguments) > 1) { r->extra_args = extra_callback_arguments[1..]; } }
1615cc2016-05-20Pontus Östlund  protected void async_fail(SessionQuery q)
51354e2016-05-13Pontus Östlund  {
1615cc2016-05-20Pontus Östlund  TRACE("fail q: %O -> %O\n", q, ::url_requested);
51354e2016-05-13Pontus Östlund  mapping ret = ([ "status" : q->status, "status_desc" : q->status_desc, "host" : q->host, "headers" : copy_value(q->headers), "url" : ::url_requested ]);
1615cc2016-05-20Pontus Östlund  TRACE("Ret: %O\n", ret);
aeb3892017-03-01Pontus Östlund  set_extra_args_in_result(ret);
51354e2016-05-13Pontus Östlund  // clear callbacks for possible garbation of this Request object con->set_callbacks(0, 0); function fc = fail_callback; set_callbacks(0, 0, 0); // drop all references extra_callback_arguments = 0; if (fc) { fc(Failure(ret)); } }
1615cc2016-05-20Pontus Östlund  protected void async_ok(SessionQuery q)
51354e2016-05-13Pontus Östlund  { TRACE("async_ok: %O -> %s!\n", q->host, ::url_requested); ::check_for_cookies(); if (con->status >= 300 && con->status < 400 && con->headers->location && follow_redirects) { Standards.URI loc = Standards.URI(con->headers->location,url_requested);
1615cc2016-05-20Pontus Östlund  TRACE("New location: %O -> %O (%O)\n", url_requested, loc, con->headers);
51354e2016-05-13Pontus Östlund  if (loc->scheme == "http" || loc->scheme == "https") {
1615cc2016-05-20Pontus Östlund  con->set_callbacks(0, 0); ::destroy(); // clear
51354e2016-05-13Pontus Östlund  follow_redirects--; do_async(prepare_method("GET", loc)); return; } } // clear callbacks for possible garbation of this Request object con->set_callbacks(0, 0); if (data_callback) {
aeb3892017-03-01Pontus Östlund  con->async_fetch(async_data); // start data downloading
51354e2016-05-13Pontus Östlund  } else { extra_callback_arguments = 0; // to allow garb } } protected void async_data() { mapping ret = ([ "host" : con->host, "status" : con->status, "status_desc" : con->status_desc, "headers" : copy_value(con->headers), "data" : con->data(), "url" : ::url_requested ]);
aeb3892017-03-01Pontus Östlund  set_extra_args_in_result(ret);
51354e2016-05-13Pontus Östlund  // clear callbacks for possible garbation of this Request object con->set_callbacks(0, 0); if (data_callback) { data_callback(Success(ret)); } extra_callback_arguments = 0; } void destroy() { TRACE("Destructor called in Request: %O\n", ::url_requested);
1615cc2016-05-20Pontus Östlund  ::set_callbacks(0, 0, 0);
51354e2016-05-13Pontus Östlund  ::destroy(); } } class SessionQuery { inherit parent::SessionQuery; protected void create() { #if constant (this::maxtime) TRACE("# Query has maxtime\n"); if (Session::maxtime) { this::maxtime = Session::maxtime; } #endif if (Session::timeout) { this::timeout = Session::timeout; } }
0b9ed42018-10-15Karl Gustav Sterneberg  // Support for streaming query (data in form of Stdio.Stream). Implemented // here in wait of support in Protocols.HTTP. protected Stdio.Stream source_stream; protected int source_stream_length; this_program async_request (string server, int port, string query, void|mapping|string headers, void|string|Stdio.Stream data) { if (objectp(data)) { set_source_stream(data); data = UNDEFINED; } if (source_stream) { if (data) error ("String data not allowed in streaming mode.\n"); if (stringp (headers)) error ("String headers not allowed in streaming mode.\n"); if (!headers) headers = ([]); headers["Content-Length"] = source_stream_length; } ::async_request (server, port, query, headers, data); } protected void async_write() { ::async_write(); if (!con->query_write_callback() && source_stream) { // Headers sent, continue with data. Stdio.sendfile (0, source_stream, 0, source_stream_length, 0, con, lambda(int bytes_sent) { source_stream->close(); source_stream = 0; }); } } void set_source_stream (Stdio.Stream s) { source_stream = s; source_stream_length = s->stat()->size; } this_program sync_request(string server, int port, string query, void|mapping|string http_headers, void|string|Stdio.Stream data) { if (objectp(data)) { // Did not bother to write a loop in order to be able to read more than // 0x7fffffff bytes from the data argument, since sending so much data // using sync request is a pretty bad idea anyway... string tmp = data->read(0x7fffffff); data->close(); data = tmp; } return ::sync_request(server, port, query, http_headers, data); } // End of code "Support for streaming query".
51354e2016-05-13Pontus Östlund  } }