Branch: Tag:

2019-06-14

2019-06-14 11:50:17 by Stephen R. van den Berg <srb@cuci.nl>

Shuffler.Shuffle: Numerous fixes and add set_wrap_callback().

The fixes include:
- Fixed broken end-of-stream detection for d_source_pikestream.c.
- Reduce memory footprint of source-structs by half.
- Fixed memory corruption issues because a memcpy was used on
overlapping regions.
- Eliminated all dynamic memory allocations in the shuffler.
For file-descriptor based streams it still has a fixed overhead of 16KB
per stream, for all other streams the overhead now is close to zero.

Still TODO:
- Reduce the memory overhead for fd-based streams by half; it uses
double buffering now, it could trivially be reduced to use a single
buffer.

29:    struct object *obj;    struct object *cb_obj;    struct pike_string *str; +  int available;       void (*when_data_cb)( void *a );    void *when_data_cb_arg;
44:   static void setup_callbacks( struct source *src )   {    struct pf_source *s = (struct pf_source *)src; -  if( !s->str ) +  if( !s->available )    {    ref_push_object( s->cb_obj );    apply( s->obj, "set_read_callback", 1 );
66:    pop_stack();   }    + static void frees(struct pf_source*s) { +  if (s->str) { +  free_string(s->str); +  s->str = 0; +  } + }      static struct data get_data( struct source *src, off_t len )   {    struct pf_source *s = (struct pf_source *)src; -  struct data res = { 0, 0, 0, NULL }; -  char *buffer = NULL; +  struct data res;    -  if( s->str ) -  { -  len = s->str->len; -  if( s->skip ) -  { -  if( s->skip >= (size_t)s->str->len ) -  { -  s->skip -= (size_t)s->str->len; -  res.len = -2; +  if (s->available) { +  if (!s->str) { +  s->s.eof = 1; +  res.len = 0;    return res;    } -  +  len = s->str->len; +  if (s->skip) { +  if (s->skip >= len) { +  frees(s); +  s->skip -= len; +  goto getmore; +  }    len -= s->skip; -  buffer = malloc( len ); -  memcpy( buffer, s->str->str+s->skip, len); +     } -  else -  { -  if( s->len > 0 ) -  { -  if( s->len < (size_t)len ) +  if (s->len) { +  if (s->len < (size_t)len)    len = s->len;    s->len -= len; -  if( !s->len ) +  if (!s->len)    s->s.eof = 1;    } -  buffer = malloc( len ); -  memcpy( buffer, s->str->str, len ); -  } -  res.data = buffer; +  res.data = s->str->str + s->skip;    res.len = len; -  res.do_free = 1; -  free_string( s->str ); -  s->str = 0; -  setup_callbacks( src ); -  } -  else if( !s->len ) +  if (s->available < 0)    s->s.eof = 1; -  else +  s->available = 0; +  } else { + getmore:    /* No data available, but there should be in the future (no EOF, nor    * out of the range of data to send as specified by the arguments to    * source_stream_make)    */    res.len = -2; -  +  } +  setup_callbacks(src);    return res;   }      static void free_source( struct source *src )   { -  +  struct pf_source *s = (struct pf_source *)src;    remove_callbacks( src ); -  free_object(((struct pf_source *)src)->cb_obj); -  free_object(((struct pf_source *)src)->obj); +  frees(s); +  free_object(s->cb_obj); +  free_object(s->obj);   }      static void f_got_data( INT32 args )
134:       remove_callbacks( (struct source *)s );    -  if( s->str || +  if (args < 2 ||    TYPEOF(Pike_sp[-1]) != PIKE_T_STRING ||    Pike_sp[-1].u.string->size_shift ||    Pike_sp[-1].u.string->len == 0)    { -  s->s.eof = 1; +  s->available = -1;    pop_n_elems(args);    push_int(0);    return;    } -  +  frees(s); +  s->available = 1;    s->str = Pike_sp[-1].u.string;    Pike_sp--;    pop_n_elems(args-1);
174:       res->len = len;    res->skip = start; +  res->available = 0;       res->s.get_data = get_data;    res->s.free_source = free_source;