pike.git / lib / modules / Sql.pmod / pgsql_util.pmod

version» Context lines:

pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:27:   #define CLOSING 5   #define CLOSED 6   #define PURGED 7      #define NOERROR 0 // Error states networkparser   #define PROTOCOLERROR 1   #define PROTOCOLUNSUPPORTED 2      #define LOSTERROR "Database connection lost"    + #if PG_DEADLOCK_SENTINEL + private multiset mutexes = set_weak_flag((<>), Pike.WEAK); + private int deadlockseq; +  + private class MUTEX { +  inherit Thread.Mutex; +  +  private Thread.Thread sentinelthread; +  +  private void dump_lockgraph(Thread.Thread curthread) { +  sleep(PG_DEADLOCK_SENTINEL); +  String.Buffer buf = String.Buffer(); +  buf.add("\n{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{\n"); +  buf.sprintf("Deadlock detected at\n%s\n", +  describe_backtrace(curthread.backtrace(), -1)); +  foreach(mutexes; this_program mu; ) +  if (mu) { +  Thread.Thread ct; +  if (ct = mu.current_locking_thread()) +  buf.sprintf("====================================== Mutex: %O\n%s\n", +  mu, describe_backtrace(ct.backtrace(), -1)); +  } +  buf.add("}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}"); +  werror(replace(buf.get(), "\n", sprintf("\n%d> ", deadlockseq++)) + "\n"); +  } +  +  Thread.MutexKey lock(void|int type) { +  Thread.MutexKey key; +  if (!(key = trylock(type))) { +  sentinelthread = Thread.Thread(dump_lockgraph, this_thread()); +  key = ::lock(type); +  sentinelthread->kill(); +  } +  return key; +  } +  +  protected void create() { +  //::create(); +  mutexes[this] = 1; +  } + }; + #else + #define MUTEX Thread.Mutex + #endif +    //! The instance of the pgsql dedicated backend.   final Pike.Backend local_backend;      private Pike.Backend cb_backend; -  + private Result qalreadyprinted;   private Thread.Mutex backendmux = Thread.Mutex();   private Thread.ResourceCount clientsregistered = Thread.ResourceCount();      constant emptyarray = ({});   constant describenodata    = (["datarowdesc":emptyarray, "datarowtypes":emptyarray,    "datatypeoid":emptyarray]);   private constant censoroptions = (<"use_ssl", "force_ssl",    "cache_autoprepared_statements", "reconnect", "text_query", "is_superuser",    "server_encoding", "server_version", "integer_datetimes",
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:336:    realbuffer->sendcmd(SENDOUT);   #endif    }    }   };      class conxiin {    inherit Stdio.Buffer:i;       final Thread.Condition fillread; -  final Thread.Mutex fillreadmux; +  final MUTEX fillreadmux;    final int procmsg;    private int didreadcb;       protected final bool range_error(int howmuch) {   #ifdef PG_DEBUG    if (howmuch <= 0)    error("Out of range %d\n", howmuch);   #endif    if (fillread) {    Thread.MutexKey lock = fillreadmux->lock();
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:371:    Thread.MutexKey lock = fillreadmux->lock();    if (procmsg && id)    procmsg = 0, lock = 0, Thread.Thread(id);    else if (fillread)    didreadcb = 1, fillread.signal();    return 0;    }       private void create() {    i::create(); -  fillreadmux = Thread.Mutex(); +  fillreadmux = MUTEX();    fillread = Thread.Condition();    }   };      class sfile {    inherit Stdio.File;    final int query_fd() {    return is_open() ? ::query_fd() : -1;    }   };      class conxion {    inherit Stdio.Buffer:o;    final conxiin i;       private Thread.Queue qportals; -  final Thread.Mutex shortmux; +  final MUTEX shortmux;    private int closenext;       final sfile   #if constant(SSL.File)    |SSL.File   #endif    socket;    private int towrite;    final multiset(Result) runningportals = (<>);    -  final Thread.Mutex nostash; +  final MUTEX nostash;    final Thread.MutexKey started;    final Thread.Queue stashqueue;    final Thread.Condition stashavail;    final Stdio.Buffer stash;    //! @ignore    final int(KEEP..SYNCSEND) stashflushmode;    //! @endignore    final Thread.ResourceCount stashcount;    final int synctransact;   #ifdef PG_DEBUGRACE
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:430:    portal._portalname, ++queueoutidx, synctransact, sizeof(this));    }       final bufcon|conxsess start(void|int waitforreal) {    Thread.MutexKey lock;   #ifdef PG_DEBUGRACE    if (nostash->current_locking_thread())    PD("Nostash locked by %s\n",    describe_backtrace(nostash->current_locking_thread()->backtrace()));   #endif -  if (lock = (waitforreal ? nostash->lock : nostash->trylock)(1)) { +  while (lock = (waitforreal ? nostash->lock : nostash->trylock)(1)) {    int mode; -  +  if (!stashcount->drained()) { +  lock = 0; // Force release before acquiring next +  stashcount->wait_till_drained(); +  continue; +  }   #ifdef PG_DEBUGRACE    conxsess sess = conxsess(this);   #endif    started = lock; -  +  lock = 0; // Force release before acquiring next    lock = shortmux->lock(); -  stashcount->wait_till_drained(lock); +     mode = getstash(KEEP);    lock = 0;    if (mode > KEEP)    sendcmd(mode); // Force out stash to the server   #ifdef PG_DEBUGRACE    return sess;   #else    return this;   #endif    }
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:480:    PD("%d>Got stash mode %d > %d\n",    socket->query_fd(), stashflushmode, mode);    if (stashflushmode > mode)    mode = stashflushmode;    stashflushmode = KEEP;    }    return mode;    }       final void sendcmd(void|int mode, void|Result portal) { -  Thread.MutexKey lock; +     if (portal)    queueup(portal);   unfinalised:    do {    switch (mode) {    default:    break unfinalised;    case SYNCSEND:    PD("%d>Sync %d %d Queue\n",    socket->query_fd(), synctransact, ++queueoutidx);    add(PGSYNC);    mode = SENDOUT;    break;    case FLUSHLOGSEND:    PD("%d>%O %d Queue simplequery %d bytes\n", socket->query_fd(),    portal._portalname, ++queueoutidx, sizeof(this));    mode = FLUSHSEND;    }    qportals->write(synctransact++);    } while (0); -  lock = shortmux->lock(); +  Thread.MutexKey lock = shortmux->lock();    mode = getstash(mode);   #ifdef PG_DEBUG    mixed err =   #endif    catch {   outer:    do {    switch (mode) {    default:    PD("%d>Skip flush %d Queue %O\n",
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:665:    }    return res;    }       private void create(proxy pgsqlsess, Thread.Queue _qportals, int nossl) {    o::create();    qportals = _qportals;    synctransact = 1;    socket = sfile();    i = conxiin(); -  shortmux = Thread.Mutex(); -  nostash = Thread.Mutex(); +  shortmux = MUTEX(); +  nostash = MUTEX();    closenext = 0;    stashavail = Thread.Condition();    stashqueue = Thread.Queue();    stash = Stdio.Buffer();    stashcount = Thread.ResourceCount();    Thread.Thread(connectloop, pgsqlsess, nossl);    }   };      #ifdef PG_DEBUGRACE
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:730:    final int _fetchlimit;    private int(0..1) alltext;    final int(0..1) _forcetext;    private int syncparse;    private int transtype;       final string _portalname;       private int inflight;    int portalbuffersize; -  private Thread.Mutex closemux; +  private MUTEX closemux;    private Thread.Queue datarows;    private Thread.ResourceCountKey stmtifkey, portalsifkey;    private array(mapping(string:mixed)) datarowdesc;    final array(int) datarowtypes; // types from datarowdesc    private string statuscmdcomplete;    private int bytesreceived;    final int _synctransact;    final Thread.Condition _ddescribe; -  final Thread.Mutex _ddescribemux; +  final MUTEX _ddescribemux;    final Thread.MutexKey _unnamedportalkey, _unnamedstatementkey;    final array _params;    final string _query;    final string _preparedname;    final mapping(string:mixed) _tprepared;    private function(:void) gottimeout;    private int timeout;       protected string _sprintf(int type) {    string res;    switch (type) {    case 'O':    int fd = -1;    if (c && c->socket)    catch(fd = c->socket->query_fd());    res = sprintf("Result state: %d numrows: %d eof: %d inflight: %d\n"    "query: %O\n"    "fd: %O portalname: %O datarows: %d"    " synctransact: %d laststatus: %s\n",    _state, index, eoffound, inflight, -  _query, fd, _portalname, +  qalreadyprinted == this ? "..." : _query, fd, _portalname,    datarowtypes && sizeof(datarowtypes), _synctransact,    statuscmdcomplete    || (_unnamedstatementkey ? "*parsing*" : "")); -  +  qalreadyprinted = this;    break;    }    return res;    }       protected void create(proxy _pgsqlsess, conxion _c, string query,    int _portalbuffersize, int alltyped, array params, int forcetext,    int _timeout, int _syncparse, int _transtype) {    pgsqlsess = _pgsqlsess;    if (c = _c)    cr = c->i;    else    losterror();    _query = query;    datarows = Thread.Queue();    _ddescribe = Thread.Condition(); -  _ddescribemux = Thread.Mutex(); -  closemux = Thread.Mutex(); +  _ddescribemux = MUTEX(); +  closemux = MUTEX();    portalbuffersize = _portalbuffersize;    alltext = !alltyped;    _params = params;    _forcetext = forcetext;    _state = PORTALINIT;    timeout = _timeout;    syncparse = _syncparse;    gottimeout = _pgsqlsess->cancelquery;    c->runningportals[this] = 1;    transtype = _transtype;
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:804:    //! Returns the command-complete status for this query.    //!    //! @seealso    //! @[Sql.Result()->status_command_complete()]    /*semi*/final string status_command_complete() {    if (!statuscmdcomplete) {    if (!datarowtypes)    waitfordescribe();    {    Thread.MutexKey lock = closemux->lock(); -  if (_fetchlimit) -  _sendexecute(_fetchlimit = 0); +  if (_fetchlimit) { +  array reflock = ({ lock }); +  lock = 0; +  _sendexecute(_fetchlimit = 0, reflock); +  } else +  lock = 0; // Force release before acquiring next    lock = _ddescribemux->lock();    if (!statuscmdcomplete)    PT(_ddescribe->wait(lock));    }    if (this) // If object already destructed, skip the next call    trydelayederror(); // since you cannot call functions anymore    else    error(LOSTERROR);    }    return statuscmdcomplete;
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:1389:    }    if (!datarowtypes) {    if (_tprepared && dontcacheprefix->match(_query))    m_delete(pgsqlsess->prepareds, _query), _tprepared = 0;    waitfordescribe();    }    if (_state >= CLOSING)    lock = _unnamedstatementkey = 0;    else {    plugbuffer->add_int16(sizeof(datarowtypes)); -  if (sizeof(datarowtypes)) +  if (sizeof(datarowtypes)) {    plugbuffer->add_ints(map(datarowtypes, readoidformat), 2); -  else if (syncparse < 0 && !pgsqlsess->wasparallelisable +  lock = 0; +  } else if (syncparse < 0 && !pgsqlsess->wasparallelisable    && !pgsqlsess->statementsinflight->drained(1)) { -  lock = pgsqlsess->shortmux->lock(); +  lock = 0;    PD("Commit waiting for statements to finish\n"); -  catch(PT(pgsqlsess->statementsinflight->wait_till_drained(lock, 1))); +  catch(PT(pgsqlsess->statementsinflight->wait_till_drained(1)));    } -  lock = 0; +     PD("Bind portal %O statement %O\n", _portalname, _preparedname);    _fetchlimit = pgsqlsess->_fetchlimit;    _bindportal();    conxsess bindbuffer = c->start();    _unnamedstatementkey = 0;    stmtifkey = 0;    CHAIN(bindbuffer)->add_int8('B')->add_hstring(plugbuffer, 4, 4);    if (!_tprepared && sizeof(_preparedname))    closestatement(CHAIN(bindbuffer), _preparedname);    _sendexecute(_fetchlimit
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:1429:    if (_tprepared) {    _tprepared.datarowdesc = datarowdesc;    _tprepared.datarowtypes = datarowtypes;    }    }       final void _parseportal() {    {    Thread.MutexKey lock = closemux->lock();    _state = PARSING; -  { -  Thread.MutexKey lockc = pgsqlsess->shortmux->lock(); +     if (syncparse || syncparse < 0 && pgsqlsess->wasparallelisable) {    PD("Commit waiting for statements to finish\n"); -  catch(PT(pgsqlsess->statementsinflight->wait_till_drained(lockc))); +  catch(PT(pgsqlsess->statementsinflight->wait_till_drained()));    }    stmtifkey = pgsqlsess->statementsinflight->acquire();    } -  } +     statuscmdcomplete = 0;    pgsqlsess->wasparallelisable = paralleliseprefix->match(_query);    }    -  final void _releasestatement(void|int nolock) { -  Thread.MutexKey lock; -  if (!nolock) -  lock = closemux->lock(); +  final void _releasestatement() { +  Thread.MutexKey lock = closemux->lock(2);    if (_state <= BOUND) {    _state = COMMITTED;    stmtifkey = 0;    }    }       final void _bindportal() {    Thread.MutexKey lock = closemux->lock();    _state = BOUND;    portalsifkey = pgsqlsess->portalsinflight->acquire();
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:1480:    switch (_state) {    case BOUND:    case PARSING:    stmtifkey = 0;    }    _state = PURGED;    }    releaseconditions();    }    -  final int _closeportal(conxsess cs) { +  final int _closeportal(conxsess cs, array(Thread.MutexKey) reflock) {    void|bufcon|conxsess plugbuffer = CHAIN(cs);    int retval = KEEP;    PD("%O Try Closeportal %d\n", _portalname, _state); -  Thread.MutexKey lock = closemux->lock(2); // When called from _sendexecute(), it is already locked +     _fetchlimit = 0; // disables further Executes    switch (_state) {    case PARSING:    case BOUND: -  _releasestatement(1); +  _releasestatement();    }    switch (_state) {    case PORTALINIT:    case PARSING:    _unnamedstatementkey = 0;    _state = CLOSING;    break;    case COPYINPROGRESS:    PD("CopyDone\n");    plugbuffer->add("c\0\0\0\4");    case COMMITTED:    case BOUND:    _state = CLOSING; -  lock = 0; +  if (reflock) +  reflock[0] = 0;    PD("Close portal %O\n", _portalname);    if (_portalname && sizeof(_portalname)) {    plugbuffer->add_int8('C')    ->add_hstring(({'P', _portalname, 0}), 4, 4);    retval = FLUSHSEND;    } else    _unnamedportalkey = 0;    portalsifkey = 0;    if (pgsqlsess->portalsinflight->drained()) {    if (plugbuffer->stashcount->drained() && transtype != TRANSBEGIN)
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:1535:       private void replenishrows() {    if (_fetchlimit && sizeof(datarows) <= _fetchlimit >> 1) {    Thread.MutexKey lock = closemux->lock();    if (_fetchlimit) {    _fetchlimit = pgsqlsess._fetchlimit;    if (bytesreceived)    _fetchlimit =    min((portalbuffersize >> 1) * index / bytesreceived || 1, _fetchlimit);    if (_fetchlimit) -  if (inflight <= (_fetchlimit - 1) >> 1) -  _sendexecute(_fetchlimit); -  else +  if (inflight <= (_fetchlimit - 1) >> 1) { +  array reflock = ({ lock }); +  lock = 0; +  _sendexecute(_fetchlimit, reflock); +  } else    PD("<%O _fetchlimit %d, inflight %d, skip execute\n",    _portalname, _fetchlimit, inflight);    }    }    }       final void _processdataready(array datarow, void|int msglen) {    bytesreceived += msglen;    inflight--;    if (_state<CLOSED)
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:1578:       final void _releasesession(void|string statusccomplete) {    c->runningportals[this] = 0;    if (statusccomplete && !statuscmdcomplete) {    Thread.MutexKey lock = _ddescribemux->lock();    statuscmdcomplete = statusccomplete;    _ddescribe->broadcast();    }    inflight = 0;    conxsess plugbuffer; +  array(Thread.MutexKey) reflock = ({closemux->lock()});    if (!catch(plugbuffer = c->start())) -  plugbuffer->sendcmd(_closeportal(plugbuffer)); +  plugbuffer->sendcmd(_closeportal(plugbuffer, reflock)); +  reflock = 0;    if (_state < CLOSED)    _state = CLOSED;    datarows->write(1); // Signal EOF    releaseconditions();    }       protected void _destruct() {    catch { // inside destructors, exceptions don't work    _releasesession();    };    }    -  final void _sendexecute(int fetchlimit, void|bufcon|conxsess plugbuffer) { +  final void _sendexecute(int fetchlimit, +  void|array(Thread.MutexKey)|bufcon|conxsess plugbuffer) {    int flushmode; -  +  array(Thread.MutexKey) reflock;    PD("Execute portal %O fetchlimit %d transtype %d\n", _portalname,    fetchlimit, transtype); -  if (!plugbuffer) +  if (arrayp(plugbuffer)) { +  reflock = plugbuffer;    plugbuffer = c->start(1); -  +  }    CHAIN(plugbuffer)->add_int8('E')->add_hstring(({_portalname, 0}), 4, 8)    ->add_int32(fetchlimit);    if (!fetchlimit) {    if (transtype != NOTRANS)    pgsqlsess.intransaction = transtype == TRANSBEGIN; -  flushmode = _closeportal(plugbuffer) == SYNCSEND +  flushmode = _closeportal(plugbuffer, reflock) == SYNCSEND    || transtype == TRANSEND ? SYNCSEND : FLUSHSEND;    } else    inflight += fetchlimit, flushmode = FLUSHSEND;    plugbuffer->sendcmd(flushmode, this);    }       inline private array setuptimeout() {    return local_backend->call_out(gottimeout, timeout);    }   
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:1759:    function(Result, array(array(mixed)), mixed ...:void) callback,    mixed ... args) {    if (callback)    Thread.Thread(run_result_array_cb, callback, args);    }      };      class proxy {    final int _fetchlimit = FETCHLIMIT; -  final Thread.Mutex unnamedportalmux; -  final Thread.Mutex unnamedstatement; +  final MUTEX unnamedportalmux; +  final MUTEX unnamedstatement;    private Thread.MutexKey termlock;    private Thread.ResourceCountKey backendreg;    final Thread.ResourceCount portalsinflight, statementsinflight;    final int(0..1) wasparallelisable;    final int(0..1) intransaction;       final conxion c;    private string cancelsecret;    private int backendpid;    final int(-128..127) backendstatus;
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:1794:    final int(0..1) invalidatecache;    private Thread.Queue qportals;    final mixed delayederror;    final function (:void) readyforquery_cb;       final string host;    final int(0..65535) port;    final string database, user, pass;    private Crypto.Hash.SCRAM SASLcontext;    final Thread.Condition waitforauthready; -  final Thread.Mutex shortmux; +  final MUTEX shortmux;    final int readyforquerycount;       private string _sprintf(int type) {    string res;    switch (type) {    case 'O':    res = sprintf(DRIVERNAME".proxy(%s@%s:%d/%s,%d,%d)",    user, host, port, database, c && c->socket && c->socket->query_fd(),    backendpid);    break;
pike.git/lib/modules/Sql.pmod/pgsql_util.pmod:1828:    this::options = options;       if (!host) host = PGSQL_DEFAULT_HOST;    if (has_value(host,":") && sscanf(host,"%s:%d", host, port) != 2)    error("Error in parsing the hostname argument\n");    this::host = host;       if (!port)    port = PGSQL_DEFAULT_PORT;    backendreg = register_backend(); -  shortmux = Thread.Mutex(); +  shortmux = MUTEX();    PD("Connect\n");    waitforauthready = Thread.Condition();    qportals = Thread.Queue();    readyforquerycount = 1;    qportals->write(1);    if (!(c = conxion(this, qportals, 0)))    error("Couldn't connect to database on %s:%d\n", host, port);    runtimeparameter = ([]); -  unnamedportalmux = Thread.Mutex(); -  unnamedstatement = Thread.Mutex(); +  unnamedportalmux = MUTEX(); +  unnamedstatement = MUTEX();    readyforquery_cb = connect_cb;    portalsinflight = Thread.ResourceCount();    statementsinflight = Thread.ResourceCount();    wasparallelisable = 0;    }       final int is_open() {    return c && c->socket && c->socket->is_open();    }