51354e2016-05-13Pontus Östlund //! A HTTP client that does async calls but waits for the result before //! returning the result. If the request takes too long the client will abort. //! The timeout can be set per request but is @tt{60@} seconds per default //! //! @example //! @code //! HTTPClient.Arguments args = HTTPClient.Arguments(([ //! "variables" : ([ "a" : 1 ]), // optional //! "maxtime" : 30 // optional //! ])); //! //! HTTPClient.Result res = HTTPClient.sync_get("http://the.url/to/fetch", args); //! //! if (res->ok) { //! werror("Call to %s was ok: %s\n", res->url, res->data[0..50]); //! } //! else { //! werror("Call to %s failed with status %d %s\n", //! res->url, res->status, res->status_description); //! } //! @endcode //! //! You can also do completly async requests with callbacks //! //! @example //! @code //! HTTPClient.Arguments args = HTTPClient.Arguments(); //! args->on_success(lambda (HTTPClient.Result resp) { //! werror("Successful request for %O\n", resp->url); //! }); //! //! args->on_failure(lambda (HTTPClient.Result resp) { //! werror("Failed request for %O (%d %s)\n", resp->url, //! resp->status, resp->status_description); //! }); //! //! HTTPClient.async_get("http://domain.com", args); //! @endcode
2612f72017-03-01Anders Johansson // #define HTTP_CLIENT_DEBUG
51354e2016-05-13Pontus Östlund  #ifdef HTTP_CLIENT_DEBUG # define TRACE(X...)werror("%s:%d: %s",basename(__FILE__),__LINE__,sprintf(X)) #else # define TRACE(X...)0 #endif protected constant DEFAULT_MAXTIME = 60; //! Do a HTTP GET request public Result sync_get(Protocols.HTTP.Session.URL uri, void|Arguments args) { return do_safe_method("GET", uri, args); } //! Do a HTTP POST request public Result sync_post(Protocols.HTTP.Session.URL uri, void|Arguments args) { return do_safe_method("POST", uri, args); } //! Do a HTTP PUT request public Result sync_put(Protocols.HTTP.Session.URL uri, void|Arguments args) { return do_safe_method("PUT", uri, args); } //! Do a HTTP DELETE request public Result sync_delete(Protocols.HTTP.Session.URL uri, void|Arguments args) { return do_safe_method("DELETE", uri, args); } //! Do an async HTTP GET request public void async_get(Protocols.HTTP.Session.URL uri, void|Arguments args) { do_safe_method("GET", uri, args, true); } //! Do an async HTTP POST request public void async_post(Protocols.HTTP.Session.URL uri, void|Arguments args) { do_safe_method("POST", uri, args, true); } //! Do an async HTTP PUT request public void async_put(Protocols.HTTP.Session.URL uri, void|Arguments args) { do_safe_method("PUT", uri, args, true); } //! Do an async HTTP DELETE request public void async_delete(Protocols.HTTP.Session.URL uri, void|Arguments args) { do_safe_method("DELETE", uri, args, true); } //! Fetch an URL with a timeout with the @[http_method] method. public Result do_safe_method(string http_method, Protocols.HTTP.Session.URL uri, void|Arguments args, void|bool async) {
ea3bea2016-05-13Pontus Östlund  if (!args) { args = Arguments(); }
51354e2016-05-13Pontus Östlund  Result res; Session s = Session(); object /* Session.Request */ qr; Thread.Queue q; mixed co_maxtime; if (args->maxtime) s->maxtime = args->maxtime; if (args->timeout) s->timeout = args->timeout; if (!async) { q = Thread.Queue(); }
1615cc2016-05-20Pontus Östlund  function(Result:void) cb = lambda (Result r) { TRACE("Got callback: %O\n", r); res = r; q && q->write("@"); if (async) { if (r->ok && args->on_success) { args->on_success(res); } else if (!r->ok && args->on_failure) { args->on_failure(r); } qr = 0; s = 0; } };
51354e2016-05-13Pontus Östlund  qr = s->async_do_method_url(http_method, uri, args->variables, args->data, args->headers,
1615cc2016-05-20Pontus Östlund  0, // headers received callback cb, // ok callback cb, // fail callback
aeb3892017-03-01Pontus Östlund  args->extra_args || ({}));
51354e2016-05-13Pontus Östlund  if (!query_has_maxtime()) {
aeb3892017-03-01Pontus Östlund  TRACE("No maxtime in Protocols.HTTP.Query. Set external max timeout: %O\n", s->maxtime || DEFAULT_MAXTIME);
51354e2016-05-13Pontus Östlund  co_maxtime = call_out(lambda () { TRACE("Timeout callback: %O\n", qr); res = Failure(([ "status" : 504, "status_desc" : "Gateway timeout", "host" : qr->con->host, "headers" : qr->con->headers, "url" : qr->url_requested ])); qr->set_callbacks(0, 0, 0, 0); qr->con->set_callbacks(0, 0, 0); destruct(qr); destruct(s); q && q->write("@"); if (async) { if (args->on_failure) { args->on_failure(res); } qr = 0; s = 0; } }, s->maxtime || DEFAULT_MAXTIME); } if (!async) { q->read(); } if (co_maxtime && co_maxtime[0]) { TRACE("Remove timeout callout\n"); remove_call_out(co_maxtime); } if (!async) { q = 0; qr = 0; s = 0; } return res; } //! If the Protocols.HTTP.Query class doesn't have the maxtime property //! we need to take care of the maxtime timeout from the outside protected int(-1..1) _query_has_maxtime = -1; protected bool query_has_maxtime() { if (_query_has_maxtime != -1) { return _query_has_maxtime == 1; } Protocols.HTTP.Query q = Protocols.HTTP.Query(); _query_has_maxtime = (int) has_index(q, "maxtime"); destruct(q);
1615cc2016-05-20Pontus Östlund  return _query_has_maxtime == 1;
51354e2016-05-13Pontus Östlund } class Arguments { //! Data fetch timeout int timeout; //! Request timeout int maxtime; //! Additional request headers mapping(string:string) headers; //! Query variables mapping(string:mixed) variables; //! POST data void|string|mapping data; //! Callback to call on successful async request function(Result:void) on_success; //! Callback to call on failed async request function(Result:void) on_failure;
aeb3892017-03-01Pontus Östlund  //! Extra arguments that will end up in the @[Result] object array(mixed) extra_args;
51354e2016-05-13Pontus Östlund  //! If @[args] is given the indices that match any of this object's //! members will set those object members to the value of the
1615cc2016-05-20Pontus Östlund  //! corresponding mapping member.
51354e2016-05-13Pontus Östlund  protected void create(void|mapping(string:mixed) args) { if (args) {
aeb3892017-03-01Pontus Östlund  foreach (args; string key; mixed value) {
51354e2016-05-13Pontus Östlund  if (has_index(this, key)) {
aeb3892017-03-01Pontus Östlund  if ((< "variables", "headers" >)[key]) { // Cast all values to string value = mkmapping(indices(value), map(values(value), lambda (mixed s) { return (string) s; })); } this[key] = value; } else { error("Unknown argument %O!\n", key);
51354e2016-05-13Pontus Östlund  } } } } } //! HTTP result class. Consider internal. //! //! @seealso //! @[Success], @[Failure] class Result { //! Internal result mapping protected mapping result; //! Return @tt{true@} if scuccess, @tt{false@} otherwise public bool `ok(); //! The HTTP response headers public mapping `headers() { return result->headers; } //! The host that was called in the request public string `host() { return result->host; } //! The HTTP status of the response, e.g @tt{200@}, @tt{201@}, @tt{404@} //! and so on. public int `status() { return result->status; } //! The textual representation of @[status]. public string `status_description() { return result->status_desc; } //! Returns the requested URL public string `url() { return result->url; }
aeb3892017-03-01Pontus Östlund  //! Extra arguments set in the @[Arguments] object. public array(mixed) `extra_args() { return result->extra_args; }
51354e2016-05-13Pontus Östlund  //! @ignore protected void create(mapping _result) { //TRACE("this_program(%O)\n", _result->headers); result = _result; } //! @endignore } //! A class representing a successful request and its response. An instance of //! this class will be given as argument to the //! @[Concurrent.Future()->on_success()] callback registered on the returned //! @[Concurrent.Future] object from @[get_url()], @[post_url()], //! @[delete_url()], @[put_url()] or @[do_method()]. class Success { inherit Result; public bool `ok() { return true; } //! The response body, i.e the content of the requested URL public string `data() { return result->data; } //! Returns the value of the @tt{content-length@} header. public int `length() { return headers && (int)headers["content-length"]; } //! Returns the content type of the requested document public string `content_type() { if (string ct = (headers && headers["content-type"])) { sscanf (ct, "%s;", ct); return ct; } } //! Returns the content encoding of the requested document, if given by the //! response headers. public string `content_encoding() { if (string ce = (headers && headers["content-type"])) { if (sscanf (ce, "%*s;%*scharset=%s", ce) == 3) { if (ce[0] == '"' || ce[0] == '\'') { ce = ce[1..<1]; } return ce; } } } } //! A class representing a failed request. An instance of //! this class will be given as argument to the //! @[Concurrent.Future()->on_failure()] callback registered on the returned //! @[Concurrent.Future] object from @[get_url()], @[post_url()], //! @[delete_url()], @[put_url()] or @[do_method()]. class Failure { inherit Result; public bool `ok() { return false; } } //! Internal class for the actual HTTP requests protected class Session { inherit Protocols.HTTP.Session : parent; public int(0..) maxtime, timeout; public int maximum_connections_per_server = 20; Request async_do_method_url(string method, URL url, void|mapping query_variables, void|string|mapping data, void|mapping extra_headers, function callback_headers_ok, function callback_data_ok, function callback_fail, array callback_arguments) {
1615cc2016-05-20Pontus Östlund  if (stringp(url)) { url = Standards.URI(url); } // Due to a bug in Protocols.HTTP.Session which is fixed in Pike 8.1 // but not yet in 8.0. (2016-05-20) if (!extra_headers || !extra_headers->host || !extra_headers->Host) { extra_headers = extra_headers || ([]);
aeb3892017-03-01Pontus Östlund  TRACE("Set host in headers: %O\n", url); if (url->scheme == "http") { extra_headers->host = url->host; if (url->port != 80) { extra_headers->host += ":" + url->port; }
1615cc2016-05-20Pontus Östlund  }
aeb3892017-03-01Pontus Östlund  else if (url->scheme == "https") { extra_headers->host = url->host; if (url->port != 443) { extra_headers->host += ":" + url->port; }
1615cc2016-05-20Pontus Östlund  } if (!sizeof(extra_headers)) { extra_headers = 0; } TRACE("Host header set?: %O\n", extra_headers); }
aeb3892017-03-01Pontus Östlund  TRACE("Request: %O\n", url);
51354e2016-05-13Pontus Östlund  return ::async_do_method_url(method, url, query_variables, data, extra_headers, callback_headers_ok, callback_data_ok, callback_fail, callback_arguments); } class Request { inherit parent::Request;
aeb3892017-03-01Pontus Östlund  protected void set_extra_args_in_result(mapping(string:mixed) r) { if (extra_callback_arguments && sizeof(extra_callback_arguments) > 1) { r->extra_args = extra_callback_arguments[1..]; } }
1615cc2016-05-20Pontus Östlund  protected void async_fail(SessionQuery q)
51354e2016-05-13Pontus Östlund  {
1615cc2016-05-20Pontus Östlund  TRACE("fail q: %O -> %O\n", q, ::url_requested);
51354e2016-05-13Pontus Östlund  mapping ret = ([ "status" : q->status, "status_desc" : q->status_desc, "host" : q->host, "headers" : copy_value(q->headers), "url" : ::url_requested ]);
1615cc2016-05-20Pontus Östlund  TRACE("Ret: %O\n", ret);
aeb3892017-03-01Pontus Östlund  set_extra_args_in_result(ret);
51354e2016-05-13Pontus Östlund  // clear callbacks for possible garbation of this Request object con->set_callbacks(0, 0); function fc = fail_callback; set_callbacks(0, 0, 0); // drop all references extra_callback_arguments = 0; if (fc) { fc(Failure(ret)); } }
1615cc2016-05-20Pontus Östlund  protected void async_ok(SessionQuery q)
51354e2016-05-13Pontus Östlund  { TRACE("async_ok: %O -> %s!\n", q->host, ::url_requested); ::check_for_cookies(); if (con->status >= 300 && con->status < 400 && con->headers->location && follow_redirects) { Standards.URI loc = Standards.URI(con->headers->location,url_requested);
1615cc2016-05-20Pontus Östlund  TRACE("New location: %O -> %O (%O)\n", url_requested, loc, con->headers);
51354e2016-05-13Pontus Östlund  if (loc->scheme == "http" || loc->scheme == "https") {
1615cc2016-05-20Pontus Östlund  con->set_callbacks(0, 0); ::destroy(); // clear
51354e2016-05-13Pontus Östlund  follow_redirects--; do_async(prepare_method("GET", loc)); return; } } // clear callbacks for possible garbation of this Request object con->set_callbacks(0, 0); if (data_callback) {
aeb3892017-03-01Pontus Östlund  con->async_fetch(async_data); // start data downloading
51354e2016-05-13Pontus Östlund  } else { extra_callback_arguments = 0; // to allow garb } } protected void async_data() { mapping ret = ([ "host" : con->host, "status" : con->status, "status_desc" : con->status_desc, "headers" : copy_value(con->headers), "data" : con->data(), "url" : ::url_requested ]);
aeb3892017-03-01Pontus Östlund  set_extra_args_in_result(ret);
51354e2016-05-13Pontus Östlund  // clear callbacks for possible garbation of this Request object con->set_callbacks(0, 0); if (data_callback) { data_callback(Success(ret)); } extra_callback_arguments = 0; } void destroy() { TRACE("Destructor called in Request: %O\n", ::url_requested);
1615cc2016-05-20Pontus Östlund  ::set_callbacks(0, 0, 0);
51354e2016-05-13Pontus Östlund  ::destroy(); } } class SessionQuery { inherit parent::SessionQuery; protected void create() { #if constant (this::maxtime) TRACE("# Query has maxtime\n"); if (Session::maxtime) { this::maxtime = Session::maxtime; } #endif if (Session::timeout) { this::timeout = Session::timeout; } } } }