Branch: Tag:

2014-11-02

2014-11-02 22:11:44 by Stephen R. van den Berg <srb@cuci.nl>

pgsql: Further finetuning, fix normal query() race condition, thread safe.

Make the result object threadsafe (e.g. calling fetch_fields() and
fetch_row() simultaneously from multiple threads on the same result
object is supported).

60:      #define ERROR(X ...) predef::error(X)    - protected int fetchlimit=FETCHLIMIT; - protected Thread.Mutex unnamedportalmux,unnamedstatement; + int _fetchlimit=FETCHLIMIT; + Thread.Mutex _unnamedportalmux; + protected Thread.Mutex unnamedstatement;   int _portalsinflight;      protected .pgsql_util.PGassist c;
73:   protected int clearmessage;   protected mapping(string:array(mixed)) notifylist=([]);   mapping(string:string) _runtimeparameter; - protected enum querystate { -  streconnect,reconnectforce,queryidle,inquery,cancelpending,canceled}; - protected querystate qstate; - protected mapping(string:mapping(string:mixed)) prepareds=([]); + mapping(string:mapping(string:mixed)) _prepareds=([]);   protected int pstmtcount;   protected int ptstmtcount; // Periodically one would like to reset this    // but checking when this is safe to do
105:   string _host;   protected string database, user, pass;   int _port; - protected multiset cachealways=(<"BEGIN","begin","END","end","COMMIT","commit">); - protected object createprefix -  =Regexp("^[ \t\f\r\n]*[Cc][Rr][Ee][Aa][Tt][Ee][ \t\f\r\n]"); - protected object dontcacheprefix -  =Regexp("^[ \t\f\r\n]*([Ff][Ee][Tt][Cc][Hh]|[Cc][Oo][Pp][Yy])[ \t\f\r\n]"); - protected object execfetchlimit -  =Regexp("^[ \t\f\r\n]*(([Uu][Pp][Dd][Aa]|[Dd][Ee][Ll][Ee])[Tt][Ee]|\ - [Ii][Nn][Ss][Ee][Rr][Tt])[ \t\f\r\n]|\ - [ \t\f\r\n][Ll][Ii][Mm][Ii][Tt][ \t\f\r\n]+[12][; \t\f\r\n]*$"); +    protected Thread.Mutex waitforauth;   protected Thread.Condition waitforauthready;   int _readyforquerycount;    - #define DERROR(msg ...) ({sprintf(msg),backtrace()}) - #define SERROR(msg ...) (sprintf(msg)) - #define USERERROR(msg) throw(msg) - #define SUSERERROR(msg ...) USERERROR(SERROR(msg)) -  +    protected string _sprintf(int type, void|mapping flags) {    string res=UNDEFINED;    switch(type) {
134:    return res;   }    - #define BOOLOID 16 - #define BYTEAOID 17 - #define CHAROID 18 - #define INT8OID 20 - #define INT2OID 21 - #define INT4OID 23 - #define TEXTOID 25 - #define OIDOID 26 - #define XMLOID 142 - #define FLOAT4OID 700 - #define FLOAT8OID 701 - #define MACADDROID 829 - #define INETOID 869 /* Force textmode */ - #define BPCHAROID 1042 - #define VARCHAROID 1043 - #define CTIDOID 1247 - #define UUIDOID 2950 -  - #define UTF8CHARSET "UTF8" - #define CLIENT_ENCODING "client_encoding" -  +    //! @decl void create()   //! @decl void create(string host, void|string database, void|string user,@   //! void|string password, void|mapping(string:mixed) options)
346:   //! This function is PostgreSQL-specific, and thus it is not available   //! through the generic SQL-interface.   void cancelquery() { -  qstate=cancelpending; +     PD("CancelRequest\n");    object lcon=getsocket(1);    lcon->add_int32(16)->add_int32(PG_PROTOCOL(1234,5678))    ->add_int32(backendpid)->add(cancelsecret)->sendcmd(flushsend);    lcon->close(); -  + #ifdef PG_DEBUGMORE +  PD("Closetrace %O\n",backtrace()); + #endif +  object plugbuffer=c->start(1); +  foreach(qportals->peek_array();;object portal) +  portal->_closeportal(plugbuffer); +  plugbuffer->sendcmd(sendout);   }      //! Changes the connection charset. When set to @expr{"UTF8"@}, the query,
461:   mapping(string:mixed) getstatistics() {    mapping(string:mixed) stats=([    "warnings_dropped":warningsdropcount, -  "current_prepared_statements":sizeof(prepareds), +  "current_prepared_statements":sizeof(_prepareds),    "current_prepared_statement_hits":totalhits,    "prepared_statement_count":pstmtcount,   #ifdef PG_STATS
538:   //! This function is PostgreSQL-specific, and thus it is not available   //! through the generic SQL-interface.   int setfetchlimit(void|int newfetchlimit) { -  int oldfetchlimit=fetchlimit; +  int oldfetchlimit=_fetchlimit;    if(!undefinedp(newfetchlimit) && newfetchlimit>=0) -  fetchlimit=newfetchlimit; +  _fetchlimit=newfetchlimit;    return oldfetchlimit;   }   
563:    return MARKSTART+(k>1?query[..k-2]:"")+MARKERROR+query[k-1..]+MARKEND;   }    - protected void reconnect_cb() { + protected void connect_cb() {    PD("%O\n",_runtimeparameter); -  if(qstate==reconnectforce) { -  lastmessage+= -  ({sprintf("Reconnected to database %s",host_info())}); -  runcallback(backendpid,"_reconnect",""); +    } - } +     - protected void processrowdescription(object portal) { -  mapping(string:mixed) tp=portal._tprepared; -  if(!tp || !tp.datarowdesc) -  Thread.Thread(dodatarows,portal); -  if(tp) -  tp.datarowdesc=portal._datarowdesc; + protected void reconnect_cb() { +  lastmessage+=({sprintf("Reconnected to database %s",host_info())}); +  runcallback(backendpid,"_reconnect","");   }      protected array(string) showbindings(object portal) {
662:    showportal(msgtype);   #endif    } -  if(qstate==cancelpending) -  qstate=canceled,sendclose(); +     int msglen=ci->read_int32();    _msgsreceived++;    _bytesreceived+=1+msglen;
717:    case 0:    PD("Ok\n");    .pgsql_util.local_backend->remove_call_out(reconnect); -  ci->gottimeout=gottimeout; +  ci->gottimeout=cancelquery;    ci->timeout=timeout;    reconnectdelay=0;    cancelsecret="";
826: Inside #if defined(PG_DEBUG)
  #ifdef PG_DEBUG    showportal(msgtype);   #endif -  portal->_closeportal(); +  portal->_purgeportal();    }    foreach(qportals->peek_array();;.pgsql_util.pgsql_result qp) {    PD("Checking portal %O %d<=%d\n",    qp._portalname,qp._synctransact,portal);    if(qp._synctransact && qp._synctransact<=portal) -  qp->_closeportal(); +  qp->_purgeportal();    }    portal=0;    _readyforquerycount--;    if(readyforquery_cb)    readyforquery_cb(),readyforquery_cb=0; -  qstate=queryidle; -  if(waitforauthready) { -  Thread.MutexKey lock=waitforauth->lock(); -  waitforauthready->broadcast(); -  waitforauthready=0; -  lock=0; -  } +  if(waitforauthready) +  destruct(waitforauthready);    break;    case '1':   #ifdef PG_DEBUG
866:   #endif    if(portal._tprepared)    portal._tprepared.datatypeoid=a; -  preparebind(portal); +  portal->_preparebind();    break;    }    case 'T': {
875: Inside #if defined(PG_DEBUG)
   int cols=ci->read_int16();    PD("<RowDescription %d columns %O\n",cols,portal._query);    msglen-=4+2; -  foreach(a=allocate(cols);int i;) { +  foreach(a=allocate(cols);int i;)   #else -  foreach(a=allocate(ci->read_int16());int i;) { +  foreach(a=allocate(ci->read_int16());int i;)   #endif -  +  {    string s=ci->read_cstring();    mapping(string:mixed) res=(["name":s]);   #ifdef PG_DEBUG
908: Inside #if defined(PG_DEBUGMORE)
  #ifdef PG_DEBUGMORE    PD("%O\n",a);   #endif -  portal._datarowdesc=a; -  processrowdescription(portal); +  portal->_processrowdesc(a);    portal=0;    break;    }
918: Inside #if defined(PG_DEBUG)
   msglen-=4;    PD("<NoData %O\n",portal._query);   #endif -  portal._datarowdesc=({}); +     portal._fetchlimit=0; // disables subsequent Executes -  processrowdescription(portal); +  portal->_processrowdesc(({}));    portal=0;    break;    }    case 'H': -  portal._datarowdesc=getcols(); +  portal->_processrowdesc(getcols());    PD("<CopyOutResponse %d %O\n",    sizeof(portal._datarowdesc),portal._query); -  processrowdescription(portal); +     break;    case '2': {    mapping tp;
940:    int tend=gethrtime();    int tstart=tp.trun;    if(tend==tstart) -  m_delete(prepareds,portal._query); +  m_delete(_prepareds,portal._query);    else {    tp.hits++;    totalhits++;
1043:    portal._datarows->write(a);    if(serror)    ERROR(serror); -  portal->_processdataready(fetchlimit); +  portal->_processdataready();    break;    }    case 's':
1098: Inside #if defined(PG_DEBUG)
  #ifdef PG_DEBUG    msglen=0;   #endif -  portal->_processdataready(fetchlimit); +  portal->_processdataready();    break;    case 'G': -  portal._datarowdesc=getcols(); +  portal->_setrowdesc(getcols());    PD("<%O CopyInResponse %d columns\n",    portal._portalname,sizeof(portal._datarowdesc));    portal._state=copyinprogress;
1261:    _delayederror=err;    if(!ci->close() && !terminating && options.reconnect)    _connectfail(); +  throw(err);   }      //! Closes the connection to the database, any running queries are
1324:    else    c->close();    c=0; -  foreach(prepareds;;mapping tp) +  foreach(_prepareds;;mapping tp)    m_delete(tp,"preparedname");    if(!options.reconnect)    return 0;
1342:    ERROR(msg+"\n");    }    _runtimeparameter=([]); -  unnamedportalmux=Thread.Mutex(); +  _unnamedportalmux=Thread.Mutex();    unnamedstatement=Thread.Mutex(); -  qstate=force?reconnectforce:streconnect; -  readyforquery_cb=reconnect_cb; +  readyforquery_cb=force?reconnect_cb:connect_cb;    _portalsinflight=0;    return 1;   }
1363:   protected void resync_cb() {    switch(backendstatus) {    case 'T':case 'E': -  foreach(prepareds;;mapping tp) { +  foreach(_prepareds;;mapping tp) {    m_delete(tp,"datatypeoid");    m_delete(tp,"datarowdesc");    }
1728:    return ret;   }    - protected int oidformat(int oid) { -  switch(oid) { -  case BOOLOID: -  case BYTEAOID: -  case CHAROID: -  case INT8OID: -  case INT2OID: -  case INT4OID: -  case TEXTOID: -  case OIDOID: -  case XMLOID: -  case MACADDROID: -  case BPCHAROID: -  case VARCHAROID: -  case CTIDOID: -  case UUIDOID: -  return 1; //binary -  } -  return 0; // text - } -  - final protected void sendclose() { - #ifdef PG_DEBUGMORE -  PD("Closetrace %O\n",backtrace()); - #endif -  object plugbuffer=c->start(1); -  foreach(qportals->peek_array();;object portal) -  portal->_closeportal(plugbuffer); -  plugbuffer->sendcmd(sendout); - } -  - protected void gottimeout() { -  sendclose();cancelquery(); - } -  +    final protected string trbackendst(int c) {    switch(c) {    case 'I': return "idle";
1787:    return trbackendst(backendstatus);   }    - final protected void closestatement(object plugbuffer,string oldprep) { -  if(oldprep) { -  PD("Close statement %s\n",oldprep); -  plugbuffer->add_int8('C')->add_hstring(({'S',oldprep,0}),4,4); + final inline void closestatement(object plugbuffer,string oldprep) { +  .pgsql_util.closestatement(plugbuffer,oldprep);   } -  +  + protected inline string int2hex(int i) { +  return String.int2hex(i);   }      final void throwdelayederror(object parent) {
1917:    if(bindings)    q = .sql_util.emulate_bindings(q, bindings, this);    } else if(forcecache==1 -  || forcecache!=0 && (sizeof(q)>=MINPREPARELENGTH || cachealways[q])) { +  || forcecache!=0 +  && (sizeof(q)>=MINPREPARELENGTH || .pgsql_util.cachealways[q])) {    object plugbuffer=c->start(); -  if(tp=prepareds[q]) { +  if(tp=_prepareds[q]) {    if(tp.preparedname) {   #ifdef PG_STATS    prepstmtused++;
1929:    && tp.tparse*FACTORPLAN>=tstart    && (undefinedp(options.cache_autoprepared_statements)    || options.cache_autoprepared_statements)) -  preparedname=PREPSTMTPREFIX+(string)pstmtcount++; +  preparedname=PREPSTMTPREFIX+int2hex(pstmtcount++);    } else {    if(totalhits>=cachedepth) -  foreach(prepareds;string ind;tp) { +  foreach(_prepareds;string ind;tp) {    int oldhits=tp.hits;    totalhits-=oldhits-(tp.hits=oldhits>>1);    if(oldhits<=1) {    closestatement(plugbuffer,tp.preparedname); -  m_delete(prepareds,ind); +  m_delete(_prepareds,ind);    }    } -  if(forcecache!=1 && createprefix->match(q)) { // Flush cache on CREATE -  invalidatecache=1; +  if(forcecache!=1 && .pgsql_util.createprefix->match(q)) { +  invalidatecache=1; // Flush cache on CREATE    tp=UNDEFINED;    } else -  prepareds[q]=tp=([]); +  _prepareds[q]=tp=([]);    }    if(invalidatecache) {    invalidatecache=0; -  foreach(prepareds;;mapping np) { +  foreach(_prepareds;;mapping np) {    closestatement(plugbuffer,np.preparedname);    m_delete(np,"preparedname");    }
1965:    portal=.pgsql_util.pgsql_result(this,c,q,    portalbuffersize,_alltyped,from,forcetext);    portal._tprepared=tp; -  qstate=inquery; +    #ifdef PG_STATS    portalsopened++;   #endif    clearmessage=1;    object plugbuffer=c;    if(forcetext) { // FIXME What happens if portals are still open? -  portal._unnamedportalkey=unnamedportalmux->lock(1); +  portal._unnamedportalkey=_unnamedportalmux->lock(1);    portal->_openportal();    _readyforquerycount++;    Thread.MutexKey lock=unnamedstatement->lock(1);
1986:    if(!sizeof(preparedname))    preparedname=    (portal._unnamedstatementkey=unnamedstatement->trylock(1)) -  ? "" : PTSTMTPREFIX+(string)ptstmtcount++; +  ? "" : PTSTMTPREFIX+int2hex(ptstmtcount++);    // Even though the protocol doesn't require the Parse command to be    // followed by a flush, it makes a VERY noticeable difference in    // performance if it is omitted; seems like a flaw in the PostgreSQL
2005: Inside #if defined(PG_STATS)
  #ifdef PG_STATS    skippeddescribe++;   #endif -  portal._datarowdesc=tp.datarowdesc; +  portal->_setrowdesc(tp.datarowdesc);    }    portal._preparedname=preparedname;    if((portal._tprepared=tp) && tp.datatypeoid) { -  mixed e=catch(preparebind(portal)); +  mixed e=catch(portal->_preparebind());    if(e && !portal._delayederror) {    if(!stringp(e))    throw(e);
2021:    return portal;   }    - protected void preparebind(object portal) { -  array dtoid=portal._tprepared.datatypeoid; -  array(string|int) paramValues=portal._params?portal._params[2]:({}); -  if(sizeof(dtoid)!=sizeof(paramValues)) -  SUSERERROR("Invalid number of bindings, expected %d, got %d\n", -  sizeof(dtoid),sizeof(paramValues)); - #ifdef PG_DEBUGMORE -  PD("ParamValues to bind: %O\n",paramValues); - #endif -  object plugbuffer=Stdio.Buffer(); -  plugbuffer->add(portal._portalname= -  (portal._unnamedportalkey=unnamedportalmux->trylock(1)) -  ? "" : PORTALPREFIX+(string)_pportalcount++ )->add_int8(0) -  ->add(portal._preparedname)->add_int8(0)->add_int16(sizeof(paramValues)); -  foreach(dtoid;;int textbin) -  plugbuffer->add_int16(oidformat(textbin)); -  plugbuffer->add_int16(sizeof(paramValues)); -  string cenc=_runtimeparameter[CLIENT_ENCODING]; -  foreach(paramValues;int i;mixed value) { -  if(undefinedp(value)) -  plugbuffer->add_int32(-1); // NULL -  else if(stringp(value) && !sizeof(value)) { -  int k=0; -  switch(dtoid[i]) { -  default: -  k=-1; // cast empty strings to NULL for non-string types -  case BYTEAOID: -  case TEXTOID: -  case XMLOID: -  case BPCHAROID: -  case VARCHAROID:; -  } -  plugbuffer->add_int32(k); -  } else -  switch(dtoid[i]) { -  case TEXTOID: -  case BPCHAROID: -  case VARCHAROID: { -  if(!value) { -  plugbuffer->add_int32(-1); -  break; -  } -  value=(string)value; -  switch(cenc) { -  case UTF8CHARSET: -  value=string_to_utf8(value); -  break; -  default: -  if(String.width(value)>8) { -  SUSERERROR("Don't know how to convert %O to %s encoding\n", -  value,cenc); -  value=""; -  } -  } -  plugbuffer->add_hstring(value,4); -  break; -  } -  default: { -  if(!value) { -  plugbuffer->add_int32(-1); -  break; -  } -  value=(string)value; -  if(String.width(value)>8) -  if(dtoid[i]==BYTEAOID) -  value=string_to_utf8(value); -  else { -  SUSERERROR("Wide string %O not supported for type OID %d\n", -  value,dtoid[i]); -  value=""; -  } -  plugbuffer->add_hstring(value,4); -  break; -  } -  case BOOLOID:plugbuffer->add_int32(1); -  do { -  int tval; -  if(stringp(value)) -  tval=value[0]; -  else if(!intp(value)) { -  value=!!value; // cast to boolean -  break; -  } else -  tval=value; -  switch(tval) { -  case 'o':case 'O': -  catch { -  tval=value[1]; -  value=tval=='n'||tval=='N'; -  }; -  break; -  default: -  value=1; -  break; -  case 0:case '0':case 'f':case 'F':case 'n':case 'N': -  value=0; -  break; -  } -  } while(0); -  plugbuffer->add_int8(value); -  break; -  case CHAROID: -  if(intp(value)) -  plugbuffer->add_hstring(value,4); -  else { -  value=(string)value; -  switch(sizeof(value)) { -  default: -  SUSERERROR( -  "\"char\" types must be 1 byte wide, got %O\n",value); -  case 0: -  plugbuffer->add_int32(-1); // NULL -  break; -  case 1: -  plugbuffer->add_hstring(value[0],4); -  } -  } -  break; -  case INT8OID: -  plugbuffer->add_int32(8)->add_int((int)value,8); -  break; -  case OIDOID: -  case INT4OID: -  plugbuffer->add_int32(4)->add_int32((int)value); -  break; -  case INT2OID: -  plugbuffer->add_int32(2)->add_int16((int)value); -  break; -  } -  } -  portal._plugbuffer=plugbuffer; -  if(portal._tprepared) -  if(portal._tprepared.datarowdesc) -  dodatarows(portal); -  else if(dontcacheprefix->match(portal._query)) // Don't cache FETCH/COPY -  m_delete(prepareds,portal._query),portal._tprepared=0; - } -  - protected void dodatarows(object portal) { -  object plugbuffer=portal._plugbuffer; -  portal._plugbuffer=0; -  { -  array a; -  plugbuffer->add_int16(sizeof(a=portal._datarowdesc)); -  foreach(a;;mapping col) -  plugbuffer->add_int16(oidformat(col.type)); -  } -  PD("Bind portal %O statement %O\n",portal._portalname,portal._preparedname); -  portal._fetchlimit=fetchlimit; -  portal->_openportal(); -  object bindbuffer=c->start(1); -  portal._unnamedstatementkey=0; -  bindbuffer->add_int8('B')->add_hstring(plugbuffer,4,4); -  if(!portal._tprepared) -  closestatement(bindbuffer,portal._preparedname); -  portal->_sendexecute(fetchlimit -  && !(cachealways[portal._query] -  || sizeof(portal._query)>=MINPREPARELENGTH && -  execfetchlimit->match(portal._query)) -  && FETCHLIMITLONGRUN,bindbuffer); - } -  +    //! This is an alias for @[big_query()], since @[big_query()] already supports   //! streaming of multiple simultaneous queries through the same connection.   //!