pike.git/ src/ builtin.cmod

Branch: Tag:

2014-08-27

2014-08-27 18:06:14 by Per Hedbor <ph@opera.com>

Added a simple I/O buffer.

This is a work in progress.

It is intended to be used as a buffer while reading and writing from files.

It is also intended as a replacement for ADT.struct (note, lowercase struct).

36:   #include <ctype.h>   #include <errno.h>   #include <math.h> + #include <arpa/inet.h>      DECLARATIONS   
2948:   /*! @module String    */    +  + /*! @class IOBuffer +  *! +  *! A buffer to use as input or buffering when doing I/O. It is +  *! similar to @[Buffer], but can only contain 8bit data and is +  *! designed for protocol parsing. It is optimized for reading from +  *! the beginning and adding to the end, and will try to minimize the +  *! amount of data copying that is done. +  *! +  *! It can also directly read from and write to filedescriptors if so +  *! desired. This can eliminate one memory copy. +  */ + PIKECLASS IOBuffer + { +  CVAR unsigned char *buffer; +  +  CVAR size_t offset; /* reading */ +  CVAR size_t len, allocated; /* writing */ +  +  CVAR struct object *sub; +  CVAR struct pike_string *str; +  CVAR char malloced, error_mode; +  CVAR INT32 locked; +  +  typedef struct IOBuffer_struct IOBuffer; +  +  static void io_unlock( IOBuffer *io ) +  { +  io->locked--; +  } +  +  static void io_lock( IOBuffer *io ) +  { +  io->locked++; +  } +  +  static void io_ensure_unlocked(IOBuffer *io) +  { +  if( io->locked ) +  Pike_error("Can not modify the buffer right now, " +  " there are active subbuffers.\n"); +  } +  +  static unsigned char *io_add_space( IOBuffer *io, int bytes, int force ) +  { +  io_ensure_unlocked(io); +  if( !io->malloced ) +  { +  /* convert to malloced buffer from a shared one. */ +  unsigned char *old = io->buffer; +  io->buffer = xalloc( io->len + bytes + 100 ); +  io->allocated = io->len + bytes + 100; +  memcpy( io->buffer, old, io->len ); +  if( io->sub ) { +  io_unlock( get_storage(io->sub,IOBuffer_program ) ); +  free_object( io->sub ); +  } +  if( io->str ) free_string( io->str ); +  io->sub = 0; +  io->str = 0; +  io->malloced = 1; +  } +  +  if( force || (io->offset > io->len>>1) ) /* more than half used up. */ +  { +  memmove( io->buffer, io->buffer+io->offset, io->len-io->offset ); +  io->len -= io->offset; +  io->offset = 0; +  } +  if( io->len + bytes > io->allocated ) +  { +  size_t new_len = io->allocated; +  do +  new_len = ((new_len+32)*2)-32; +  while( new_len < io->len + bytes ); +  io->buffer = xrealloc( io->buffer, new_len ); +  io->allocated = new_len; +  } +  return io->buffer+io->len; +  } +  +  static void io_range_error( IOBuffer *io, int howmuch ) +  { +  /* throw error if so desired. */ +  if( io->error_mode ) +  Pike_error("Trying to read %d outside allowed range\n", howmuch); +  } +  +  static int io_avail( IOBuffer *io, int len ) +  { +  if( len < 0 || len + io->offset > io->len ) +  { +  io_range_error( io, len ); +  return 0; +  } +  return 1; +  } +  +  static void io_append( IOBuffer *io, void *p, int bytes ) +  { +  memcpy( io_add_space( io, bytes, 0 ), p, bytes ); +  io->len += bytes; +  } +  +  static int io_read( IOBuffer *io, void *to, int len ) +  { +  if( !io_avail(io,len)) +  return 0; +  memcpy( to, io->buffer+io->offset, len ); +  io->offset += len; +  return len; +  } +  +  static void io_read_and_push_string( IOBuffer *io, size_t len ) +  { +  struct pike_string *s; +  if( !io_avail(io,len)) +  { +  push_int(0); +  return; +  } +  if( len > 0x7fffffff ) +  Pike_error("This buffer is too large to convert to a string.\n"); +  s = begin_shared_string( len ); +  io_read( io, s->str, s->len ); +  push_string( end_shared_string(s) ); +  } +  +  static void io_read_and_push_buffer( IOBuffer *io, size_t len, int do_copy ) +  { +  struct pike_string *s; +  struct object *b; +  IOBuffer *to; +  if( !io_avail(io,len)) +  { +  push_int(0); +  return; +  } +  b = low_clone( IOBuffer_program ); +  to = get_storage(b,IOBuffer_program); +  push_object(b); +  +  io_lock( io ); +  +  to->buffer = io->buffer+io->offset; +  to->len = len; +  to->sub = Pike_fp->current_object; +  to->sub->refs++; +  +  if( do_copy ) +  io_add_space( to, 0, 0 );/* copies data. */ +  } +  +  static int io_read_byte_uc( IOBuffer *io ) +  { +  return io->buffer[io->offset++]; +  } +  + #if 0 +  static int io_read_byte( IOBuffer *io ) +  { +  if( !io_avail(io,1)) +  return -1; +  return io_read_byte_uc( io ); +  } + #endif +  +  static LONGEST io_read_number( IOBuffer *io, int len ) +  { +  int i; +  LONGEST res; +  +  if( !io_avail(io, len) ) +  return -1; +  +  res = 0; +  for( i=0; i<len; i++ ) +  { +  res <<= 8; +  res |= io_read_byte_uc(io); +  } +  return res; +  } +  +  +  PIKEFUN int _size_object( ) +  { +  RETURN THIS->malloced ? THIS->allocated : 0; +  } +  + /* PIKEFUN void add( object(String.IOBuffer) x )*/ + /* PIKEFUN void add( object(String.Buffer) x ) */ + /* PIKEFUN void add( System.Memory x, int len ) */ +  +  +  /*! @decl void add( string(8bit) data ) +  *! +  *! Adds the specified data to the buffer. +  */ +  PIKEFUN void add( string x ) +  { +  if( x->size_shift ) Pike_error("IOBuffer only handles 8bit data\n"); +  io_append( THIS, x->str, x->len ); +  } +  +  /*! @decl void add_byte( int(0..255) ) +  *! @decl void add_int8( int(0..255) ) +  *! Adds a single byte to the buffer. +  */ +  PIKEFUN void add_byte( int(0..255) i ) +  { +  unsigned char a = i&255; +  io_append( THIS, &a, 1 ); +  } +  PIKEFUN void add_int8( int(0..255) i ) +  { +  unsigned char a = i&255; +  io_append( THIS, &a, 1 ); +  } +  +  /*! @decl void add_int16( int(0..65535) ) +  *! @decl void add_short( int(0..65535) ) +  *! Adds a two byte value to the buffer +  */ +  PIKEFUN void add_int16( int(0..65535) i ) +  { +  unsigned short a = htons((i&65535)); +  io_append( THIS, &a, 2 ); +  } +  PIKEFUN void add_short( int(0..65535) i ) +  { +  unsigned short a = htons((i&65535)); +  io_append( THIS, &a, 2 ); +  } +  +  /*! @decl void add_int32( int i ) +  *! Adds a four byte value to the buffer +  */ +  PIKEFUN void add_int32( int i ) +  { +  INT32 a = htonl(i); +  io_append( THIS, &a, 4 ); +  } +  +  /* fixme: io_add_bignum()? +  +  This one only handles integers with size up to INT_TYPE. +  So, on 64 bit 64 bit integers, on 32 32 etc. +  */ +  static void io_add_int( IOBuffer *io, int i, int bytes ) +  { +  unsigned char tmp[SIZEOF_INT_TYPE]; +  int n = 0; +  while( bytes > SIZEOF_INT_TYPE ) +  { +  io_append(io,&n,1); +  bytes--; +  } +  for(; n<bytes; n++ ) +  { +  tmp[bytes-n] = i&255; +  i>>=8; +  } +  io_append( io, tmp, bytes ); +  } +  +  /*! @decl void add_hstring( string(0..255) data, int size_size ) +  *! Adds length/data for @[data] to the buffer. +  *! +  *! This is similar to @[sprintf("%"+size_size+"H",data)] but faster. +  */ +  PIKEFUN void add_hstring( string str, int size_size ) +  { +  if( str->size_shift ) +  Pike_error("Only 8bit is supported\n"); +  +  /* We know this is safe for len>=4, since pike strings are at +  * most 0x7ffffff bytes long. +  */ +  if( size_size < 4 && +  str->len > (1<<(8*(size_size<0?-size_size:size_size)))-1 ) +  Pike_error("Too long string\n"); +  +  io_add_int( THIS, str->len, size_size ); +  io_append( THIS, str->str, str->len ); +  } +  +  /*! @decl void add_integer( int i, int(0..) width ) +  *! +  *! Adds a generic integer to the buffer as an (width*8)bit +  *! network byteorder number. +  *! +  *! The maximum integer size really supported directly is the +  *! native integer size on your computer (the other bytes will be +  *! set to 0). +  *! +  *! Use @[sprintf] if larger numbers are desired. +  */ +  PIKEFUN void add_integer( int i, int width ) +  { +  Pike_sp-=2; +  io_add_int( THIS, i, width ); +  } +  +  /*! @decl int `[](int off) +  *! +  *! Return the character at the specified offset. +  */ +  PIKEFUN int `[]( int off ) +  flags ID_PROTECTED; +  { +  pop_stack(); +  if( off < 0 ) +  off = (THIS->len-THIS->offset)-off; +  +  if( io_avail( THIS, off ) ) +  push_int( THIS->buffer[THIS->offset+off] ); +  else +  push_int(-1); +  } +  +  PIKEFUN int `[]=( int off, int val ) +  flags ID_PROTECTED; +  { +  io_add_space( THIS, 0, 0 ); +  +  if( off < 0 ) off = (THIS->len-THIS->offset)-off; +  +  if( io_avail( THIS, off ) ) +  { +  THIS->buffer[THIS->offset+off]=(val&0xff); +  } +  else +  { +  /* hm, well. We could extend the buffer? */ +  push_int(-1); +  } +  } +  +  +  /*! @decl int _sizeof() +  *! +  *! returns the remaining buffer size, in bytes. +  */ +  PIKEFUN int _sizeof() +  flags ID_PROTECTED; +  { +  size_t l = THIS->len; +  l -= THIS->offset; +  push_int64(l); +  } +  +  PIKEFUN string cast(string to) +  { +  if( to != literal_string_string ) +  { +  push_undefined(); +  return; +  } +  if( THIS->len - THIS->offset > 0x7fffffff ) +  Pike_error("This buffer is too large to convert to a string.\n"); +  push_string(make_shared_binary_string((void*)(THIS->buffer+THIS->offset), +  (INT32)(THIS->len-THIS->offset))); +  } +  +  PIKEFUN string _sprintf(int o, mapping ignore) +  flags ID_PROTECTED; +  { +  pop_n_elems(args); +  if( o == 'O' ) +  { +  push_text("IOBuffer(%d (%d %s, @%d))"); +  push_int(THIS->len); +  push_int(THIS->allocated); +  push_text( (THIS->str ? "string" : THIS->sub ? "subbuffer" : "allocated" ) ); +  push_int(THIS->offset); +  f_sprintf(5); +  } +  else +  push_undefined(); +  } +  +  /*! @decl string(8bit) read_hstring( int n ) +  *! +  *! Identical in functionality to @[extract("%"+n+"H")] or +  *! @[read](@[read_number](@[n])) but faster. +  *! +  *! Read a network byte order number of size n*8 bits, then return the +  *! indicated number of bytes as a string. +  *! +  *! If there is not enough data available return 0. +  */ +  PIKEFUN string(0..255) read_hstring( int(0..8) bytes ) +  { +  LONGEST len; +  len = io_read_number( THIS, bytes ); +  if( len >= 0 ) +  io_read_and_push_string( THIS, len ); +  else +  push_int(0); +  } +  +  /*! @decl string(8bit) read_hbuffer( int n ) +  *! @decl string(8bit) read_hbuffer( int n, bool copy ) +  *! +  *! Same as @[read_hstring], but returns the result as an IOBuffer. +  *! +  *! No data is copied unless @[copy] is specified and true, the new buffer +  *! points into the old one. +  *! +  *! @note +  *! As long as the subbuffer exists no data can be added to the main buffer. +  *! +  *! Usually this is OK, since it often represents something that +  *! should be parsed before the next whatever is extracted from +  *! the buffer, but do take care. +  *! +  *! If you need to unlink the new buffer after it has been +  *! created, call @[trim] in it. +  */ +  PIKEFUN IOBuffer read_hbuffer( int(0..8) bytes, int|void copy ) +  { +  LONGEST len = io_read_number( THIS, bytes ); +  int do_copy = 0; +  if( copy ) +  do_copy = copy->u.integer; +  pop_n_elems(args); +  if( len >= 0 ) +  io_read_and_push_buffer( THIS, len, do_copy ); +  else +  push_int(0); +  } +  +  /*! @decl string(8bit) read_buffer( int n ) +  *! @decl string(8bit) read_buffer( int n, bool copy ) +  *! +  *! Same as @[read], but returns the result as an IOBuffer. +  *! +  *! No data is copied unless @[copy] is specified and true, the new buffer +  *! points into the old one. +  *! +  *! @note +  *! As long as the subbuffer exists no data can be added to the main buffer. +  *! +  *! Usually this is OK, since it often represents something that +  *! should be parsed before the next whatever is extracted from +  *! the buffer, but do take care. +  */ +  PIKEFUN IOBuffer read_buffer( int bytes, int|void copy ) +  { +  int do_copy = 0; +  if( copy ) +  do_copy = copy->u.integer; +  pop_n_elems(args); +  io_read_and_push_buffer( THIS, bytes, do_copy ); +  } +  +  /*! @decl int sprintf(strict_sprintf_format format, sprintf_args ... args) +  *! +  *! Appends the output from @[sprintf] at the end of the buffer. +  *! +  *! This is somewhat faster than add(sprintf(...)) since no +  *! intermediate string is created. +  */ +  PIKEFUN int sprintf(mixed ... ignored) +  rawtype tFuncV(tAttr("strict_sprintf_format", tOr(tStr, tObj)), +  tAttr("sprintf_args", tMix), tStr); +  { +  ONERROR _e; +  struct string_builder tmp; +  init_string_builder(&tmp,0); +  SET_ONERROR(_e, free_string_builder, &tmp); +  low_f_sprintf(args,0, &tmp ); +  if( tmp.s->size_shift ) +  Pike_error("IOBuffer only handles 8bit data\n"); +  io_append( THIS, tmp.s->str, tmp.s->len ); +  CALL_AND_UNSET_ONERROR(_e); +  } +  +  /*! @decl void clear() +  *! +  *! Clear the buffer. +  */ +  PIKEFUN int(0..) clear( ) +  { +  IOBuffer *io = THIS; +  io->offset = io->len = 0; +  } +  +  /*! @decl void trim() +  *! +  *! Frees unused memory. +  *! +  *! Note that calling this function excessively will slow things +  *! down, since the data often has to be copied. +  *! +  *! @note +  *! This function could possibly throw an out-of-memory error +  *! if the realloc fails to find a new (smaller) memory area. +  */ +  PIKEFUN void trim( ) +  { +  IOBuffer *io = THIS; +  io_add_space( io, 0, 1 ); +  if( io->allocated > io->len+32 ) +  { +  io->buffer = xrealloc( io->buffer, io->len ); +  io->allocated = io->len; +  } +  } +  +  /*! @decl int(0..) unread( int(0..) n ) +  *! +  *! Rewind the buffer @[n] bytes. +  *! +  *! @returns +  *! +  *! This function returns how many more bytes of buffer is +  *! available to rewind. +  *! +  *! @note +  *! +  *! Unless you add new data to the buffer using any of the add +  *! functions one you can always rewind. +  */ +  PIKEFUN int(0..) unread( int(0..) bytes ) +  { +  IOBuffer *io = THIS; +  pop_stack(); +  if( bytes < 0 || (io->offset < (unsigned)bytes) ) +  { +  io_range_error(io,-bytes); +  push_undefined(); +  return; +  } +  io->offset -= bytes; +  push_int( io->offset ); +  } +  +  /*! @decl string(8bit) read( int n ) +  *! +  *! Read @[bytes] bytes of data from the buffer. +  *! +  *! If there is not enough data available this returns 0. +  */ +  PIKEFUN string(0..255) read( int(0..) bytes ) +  { +  io_read_and_push_string(THIS, bytes ); +  } +  +  /*! @decl string(8bit) read( ) +  *! +  *! Read all data from the buffer. +  *! +  *! If there is not enough data available this returns 0. +  *! +  *! This is basically equivalent to (string)buffer, but it also +  *! removes the data from the buffer. +  */ +  PIKEFUN string(0..255) read() +  { +  io_read_and_push_string(THIS, THIS->len-THIS->offset); +  } +  +  /*! @decl read_int( int n ) +  *! +  *! Read a network (if n is positive) or little endian (if n is +  *! negative) byte order unsigned number of size n*8 bits, then +  *! return it. +  *! +  *! Will return -1 if there is not enough space available. +  */ +  PIKEFUN int read_int( int len ) +  { +  int i; +  IOBuffer *io = THIS; +  pop_stack(); +  if( !io_avail( io, len ) ) +  { +  push_int(-1); +  return; +  } +  +  if( len < 5 && len > -5 ) +  { +  LONGEST res = io_read_number( io, len ); +  if( res >= 0 ) +  { +  push_longest( res ); +  return; +  } +  } +  +  /* bignum edition ... */ +  if( len > 0 ) +  { +  push_int(0); +  for( i=0; i<len; i++ ) +  { +  push_int(8); +  o_lsh(); +  push_int(io_read_byte_uc(io)); +  o_or(); +  } +  } +  else +  { +  for( i=0; i<-len; i++ ) +  { +  push_int(io_read_byte_uc(io)); +  push_int(8*i); +  o_lsh(); +  o_or(); +  } +  } +  } +  +  /*! @decl void create( string(8bit) x ) +  *! +  *! Create a new buffer with the contents of the given +  *! string. +  *! +  *! @note +  *! This will not copy the string data, instead data will +  *! be read from the string until it needs to be modified, so the +  *! buffer creation is fast regardless of the length of the +  *! string. +  */ +  PIKEFUN void create( string x ) +  flags ID_PROTECTED; +  { +  IOBuffer *this = THIS; +  if( this->buffer ) +  Pike_error("Can not initialize twice.\n"); +  if( x->size_shift ) +  Pike_error("Only string(0..255) supported.\n"); +  this->buffer = (unsigned char*)x->str; +  this->len = x->len; +  this->malloced = 0; +  this->str = x; +  x->refs++; +  } +  +  /*! @decl void create( int|void len ) +  *! +  *! Create a new buffer of the given size. +  *! The buffer will grow if needed, so the length is only the +  *! initial size. +  *! +  *! The default initial size is 4K +  */ +  PIKEFUN void create( int len ) +  flags ID_PROTECTED; +  { +  IOBuffer *this = THIS; +  if( this->buffer ) +  Pike_error("Can not initialize twice.\n"); +  this->buffer = xalloc(len); +  this->allocated = len; +  this->malloced = 1; +  } +  +  PIKEFUN void create( ) +  flags ID_PROTECTED; +  { +  IOBuffer *this = THIS; +  if( this->buffer ) +  Pike_error("Can not initialize twice.\n"); +  this->buffer = xalloc(4096-32); +  this->allocated = 4096-32; +  this->malloced = 1; +  } +  +  +  INIT { +  IOBuffer *this = THIS; +  memset( this, 0, sizeof(IOBuffer)); +  } +  +  EXIT { +  IOBuffer *this = THIS; +  if( this->sub ) +  { +  free_object( this->sub ); +  io_unlock( get_storage(this->sub,IOBuffer_program ) ); +  } +  if( this->str ) +  free_string( this->str ); +  if( this->malloced ) +  free( this->buffer ); +  } +  + } +  +  +    /*! @class Buffer    *! A buffer, used for building strings. It's    *! conceptually similar to a string, but speed-optimised for growing