0a3d342011-11-10Arne Goedeke #include <stdlib.h>
67795d2011-11-11Arne Goedeke #include <string.h>
3854832011-11-18Arne Goedeke #include <stdint.h>
0a3d342011-11-10Arne Goedeke 
a60c242011-12-25Arne Goedeke #include "bitvector.h"
6fc6ce2011-12-30Arne Goedeke #ifdef PMOD_EXPORT #include "pike_error.h" #else #define PMOD_EXPORT #include <stdio.h> #include <unistd.h> void _Pike_error(int line, char * file, char * msg) { fprintf(stderr, "%s:%d\t%s\n", file, line, msg); _exit(1); } #define Pike_error(x) _Pike_error(__LINE__, __FILE__, x);
e90c492011-11-27Arne Goedeke #endif
6fc6ce2011-12-30Arne Goedeke #ifdef HAS___BUILTIN_EXPECT #define likely(x) (__builtin_expect((x), 1)) #define unlikely(x) (__builtin_expect((x), 0)) #else #define likely(x) (x) #define unlikely(x) (x) #endif #include "block_allocator.h" //#define BA_SEGREGATE
a5e1b82011-11-09Arne Goedeke  struct ba_page { char * data;
6fc6ce2011-12-30Arne Goedeke  ba_page_t next, prev; ba_page_t hchain;
3854832011-11-18Arne Goedeke #ifdef BA_SEGREGATE
6fc6ce2011-12-30Arne Goedeke  ba_block_t blocks_used, first;
3854832011-11-18Arne Goedeke #else
6fc6ce2011-12-30Arne Goedeke  ba_block_t blocks_used;
67795d2011-11-11Arne Goedeke  uint16_t free_mask;
a5e1b82011-11-09Arne Goedeke  uintptr_t mask[1];
3854832011-11-18Arne Goedeke #endif }; struct ba_block_header {
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG uint32_t magic; #endif #ifdef BA_SEGREGATE #include "block_allocator.h" ba_block_t next; #endif
0a3d342011-11-10Arne Goedeke };
6fc6ce2011-12-30Arne Goedeke  #define BA_MARK_FREE 0xF4337575 #define BA_MARK_ALLOC 0x4110C375 #define BA_ALLOC_INITIAL 4 #define BA_HASH_THLD 4 #ifdef BA_HASH_THLD #define IF_HASH(x) do { if(a->allocated <= BA_HASH_THLD) break; x; }\ while(0) #else #define IF_HASH(x) x
3854832011-11-18Arne Goedeke #endif
0a3d342011-11-10Arne Goedeke 
a5e1b82011-11-09Arne Goedeke 
fdfd282011-12-25Arne Goedeke size_t _malloc_counter = 0;
67795d2011-11-11Arne Goedeke static inline void ba_htable_insert(const struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  const void * ptr, const ba_page_t n);
3854832011-11-18Arne Goedeke static inline void ba_remove_page(struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  ba_page_t n); typedef struct ba_block_header * ba_block_header;
67795d2011-11-11Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke #define BA_BLOCKN(a, p, n) ((ba_block_header)(((p)->data) + (n)*((a)->block_size)))
e90c492011-11-27Arne Goedeke #define BA_LASTBLOCK(a, p) BA_BLOCKN(a, p, (a)->blocks - 1) #define BA_NBLOCK(a, p, ptr) ((uintptr_t)((char*)ptr - (p)->data)/(a)->block_size)
6fc6ce2011-12-30Arne Goedeke #define BA_DIVIDE(a, b) ((a) / (b) + (!!((a) & ((b)-1))))
a5e1b82011-11-09Arne Goedeke #define BA_PAGESIZE(a) ((a)->blocks * (a)->block_size)
3854832011-11-18Arne Goedeke #define BA_SPAGE_SIZE(a) (sizeof(struct ba_page) + (BA_DIVIDE((a)->blocks, sizeof(uintptr_t)*8) - 1)*sizeof(uintptr_t)) #define BA_HASH_MASK(a) (((a->allocated)) - 1)
6fc6ce2011-12-30Arne Goedeke #define BA_CHECK_PTR(a, p, ptr) ((char*)ptr >= p->data && (void*)BA_LASTBLOCK(a,p) >= ptr)
3854832011-11-18Arne Goedeke  #ifdef BA_SEGREGATE # define BA_PAGE(a, n) ((a)->pages + (n) - 1)
6fc6ce2011-12-30Arne Goedeke # define BA_BYTES(a) ( (sizeof(struct ba_page) + sizeof(ba_page_t)) * ((a)->allocated) )
3854832011-11-18Arne Goedeke #else # define BA_PAGE(a, n) ((ba_page)((char*)(a)->pages + ((n)-1) * ((a)->ba_page_size)))
6fc6ce2011-12-30Arne Goedeke # define BA_BYTES(a) (( (a)->ba_page_size + sizeof(ba_page_t)) * ((a)->allocated))
3854832011-11-18Arne Goedeke # define BA_LAST_BITS(a) ((a)->blocks & (sizeof(uintptr_t)*8 - 1))
6fc6ce2011-12-30Arne Goedeke # define BA_LAST_MASK(a) (BA_LAST_BITS(a) ? MASK(uintptr_t, BA_LAST_BITS(a)) : ~((uintptr_t)0))
3854832011-11-18Arne Goedeke # define BA_MASK_NUM(a) (((a)->ba_page_size - sizeof(struct ba_page) + sizeof(uintptr_t))/sizeof(uintptr_t)) #endif
67795d2011-11-11Arne Goedeke  static inline void ba_realloc(struct block_allocator * a) {
4750752011-11-19Arne Goedeke  unsigned int i;
6fc6ce2011-12-30Arne Goedeke  void *old; ba_page p; a->pages = realloc(old = a->pages, BA_BYTES(a));
67795d2011-11-11Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  if (unlikely(!a->pages)) { a->pages = old; fprintf(stderr, "realloc(%lu) failed.\n", BA_BYTES(a));
67795d2011-11-11Arne Goedeke  Pike_error("no mem"); } //fprintf(stderr, "realloc to size %u\n", a->allocated);
3854832011-11-18Arne Goedeke #ifndef BA_SEGREGATE memset((void*)BA_PAGE(a, a->num_pages+1), 0, BA_BYTES(a) - a->ba_page_size*a->num_pages); #else memset((void*)BA_PAGE(a, a->num_pages+1), 0, BA_BYTES(a) - sizeof(struct ba_page)*a->num_pages); #endif
6fc6ce2011-12-30Arne Goedeke  IF_HASH( a->htable = (ba_page_t*) BA_PAGE(a, (a->allocated)+1); /* memset(a->htable, 0, a->allocated * sizeof(ba_page_t)); */ for (i = 0; i < a->num_pages; i++) { p = BA_PAGE(a, i+1); p->hchain = 0; ba_htable_insert(a, BA_LASTBLOCK(a, p), i+1); } ); #ifdef BA_DEBUG ba_check_allocator(a, "realloc", __FILE__, __LINE__); #endif
67795d2011-11-11Arne Goedeke }
a5e1b82011-11-09Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke PMOD_EXPORT void ba_init(struct block_allocator * a, uint32_t block_size, ba_page_t blocks) {
4750752011-11-19Arne Goedeke  uint32_t page_size = block_size * blocks;
3854832011-11-18Arne Goedeke  if (blocks > 0xfffe) { Pike_error("number of blocks cannot exceed 2^16-1\n"); }
a5e1b82011-11-09Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  a->first = 0;
67795d2011-11-11Arne Goedeke  a->last_free = 0;
a5e1b82011-11-09Arne Goedeke 
0a3d342011-11-10Arne Goedeke  if ((page_size & (page_size - 1)) == 0)
67795d2011-11-11Arne Goedeke  a->magnitude = (uint16_t)ctz32(page_size);
a5e1b82011-11-09Arne Goedeke  else
67795d2011-11-11Arne Goedeke  a->magnitude = (uint16_t)ctz32(round_up32(page_size)); //fprintf(stderr, "page_size: %u, magnitude: %u\n", page_size, a->magnitude);
a5e1b82011-11-09Arne Goedeke 
0a3d342011-11-10Arne Goedeke  a->block_size = block_size;
a5e1b82011-11-09Arne Goedeke  a->blocks = blocks; a->num_pages = 0;
9387262011-12-25Arne Goedeke  a->empty_pages = 0; a->max_empty_pages = 3;
a5e1b82011-11-09Arne Goedeke 
67795d2011-11-11Arne Goedeke  // we start with management structures for 16 pages
6fc6ce2011-12-30Arne Goedeke  a->allocated = BA_ALLOC_INITIAL;
67795d2011-11-11Arne Goedeke  a->pages = NULL;
3854832011-11-18Arne Goedeke #ifndef BA_SEGREGATE a->ba_page_size = BA_SPAGE_SIZE(a); #endif
67795d2011-11-11Arne Goedeke  ba_realloc(a); }
6fc6ce2011-12-30Arne Goedeke PMOD_EXPORT void ba_free_all(struct block_allocator * a) {
4750752011-11-19Arne Goedeke  unsigned int i;
67795d2011-11-11Arne Goedeke  for (i = 0; i < a->num_pages; i++) {
4750752011-11-19Arne Goedeke  free(BA_PAGE(a, i+1)->data);
6fc6ce2011-12-30Arne Goedeke  BA_PAGE(a, i+1)->data = NULL;
a5e1b82011-11-09Arne Goedeke  }
6fc6ce2011-12-30Arne Goedeke  IF_HASH( memset(a->htable, 0, a->allocated * sizeof(ba_page_t)); );
67795d2011-11-11Arne Goedeke  a->num_pages = 0;
6fc6ce2011-12-30Arne Goedeke  a->empty_pages = 0; a->first = 0;
67795d2011-11-11Arne Goedeke }
6fc6ce2011-12-30Arne Goedeke PMOD_EXPORT void ba_count_all(struct block_allocator * a, size_t *num, size_t *size) {
4750752011-11-19Arne Goedeke  unsigned int i;
3854832011-11-18Arne Goedeke  size_t n = 0;
e90c492011-11-27Arne Goedeke  fprintf(stderr, "page_size: %u, pages: %u\n", BA_PAGESIZE(a), a->num_pages);
3854832011-11-18Arne Goedeke  *size = BA_BYTES(a) + a->num_pages * BA_PAGESIZE(a); for (i = 0; i < a->num_pages; i++) {
4750752011-11-19Arne Goedeke  n += BA_PAGE(a, i+1)->blocks_used;
3854832011-11-18Arne Goedeke  } *num = n; }
6fc6ce2011-12-30Arne Goedeke PMOD_EXPORT void ba_destroy(struct block_allocator * a) {
67795d2011-11-11Arne Goedeke  ba_free_all(a); free(a->pages);
6fc6ce2011-12-30Arne Goedeke  a->allocated = 0;
67795d2011-11-11Arne Goedeke  a->pages = NULL;
a5e1b82011-11-09Arne Goedeke }
67795d2011-11-11Arne Goedeke static inline void ba_grow(struct block_allocator * a) {
3854832011-11-18Arne Goedeke  if (a->allocated) {
6fc6ce2011-12-30Arne Goedeke  // try to detect 32bit overrun? if (a->allocated >= ((ba_page_t)1 << (sizeof(ba_page_t)*8-1))) { Pike_error("too many pages.\n"); }
3854832011-11-18Arne Goedeke  a->allocated *= 2; ba_realloc(a); } else { ba_init(a, a->block_size, a->blocks); }
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG ba_check_allocator(a, "ba_grow", __FILE__, __LINE__); #endif
67795d2011-11-11Arne Goedeke } static inline void ba_shrink(struct block_allocator * a) { a->allocated /= 2; ba_realloc(a); }
6fc6ce2011-12-30Arne Goedeke #define MIX(t) do { \ t ^= (t >> 20) ^ (t >> 12); \ t ^= (t >> 7) ^ (t >> 4); \ } while(0)
a5e1b82011-11-09Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke static inline ba_page_t hash1(const struct block_allocator * a,
67795d2011-11-11Arne Goedeke  const void * ptr) { uintptr_t t = ((uintptr_t)ptr) >> a->magnitude;
a5e1b82011-11-09Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  MIX(t); return (ba_page_t) t;
a5e1b82011-11-09Arne Goedeke }
6fc6ce2011-12-30Arne Goedeke static inline ba_page_t hash2(const struct block_allocator * a,
67795d2011-11-11Arne Goedeke  const void * ptr) { uintptr_t t = ((uintptr_t)ptr) >> a->magnitude;
a5e1b82011-11-09Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  t++; MIX(t); return (ba_page_t) t; } void ba_print_htable(const struct block_allocator * a) { unsigned int i; ba_page p; fprintf(stderr, "allocated: %u\n", a->allocated); fprintf(stderr, "num_pages: %u\n", a->num_pages); fprintf(stderr, "max_empty_pages: %u\n", a->max_empty_pages); fprintf(stderr, "empty_pages: %u\n", a->empty_pages); fprintf(stderr, "magnitude: %u\n", a->magnitude); fprintf(stderr, "block_size: %u\n", a->block_size); for (i = 0; i < a->allocated; i++) { ba_page_t n = a->htable[i]; ba_page_t hval; void * ptr; while (n) { p = BA_PAGE(a, n); ptr = BA_LASTBLOCK(a, p); hval = hash1(a, ptr); fprintf(stderr, "%u\t%X\t%p-%p\t%X (page %d)\n", i, hval, p->data, ptr, (unsigned int)((uintptr_t)ptr >> a->magnitude), n); n = p->hchain; } }
a5e1b82011-11-09Arne Goedeke } /* * insert the pointer to an allocated page into the * hashtable. uses linear probing and open allocation. */
67795d2011-11-11Arne Goedeke static inline void ba_htable_insert(const struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  const void * ptr, const ba_page_t n) { ba_page_t hval = hash1(a, ptr); ba_page_t * b = a->htable + (hval & BA_HASH_MASK(a)); #ifdef BA_DEBUG while (*b) { if (*b == n) { fprintf(stderr, "inserting (%p, %d) twice\n", ptr, n); fprintf(stderr, "is (%p, %d)\n", BA_PAGE(a, n)->data, n); Pike_error("double insert.\n"); return; } b = &BA_PAGE(a, *b)->hchain; } b = a->htable + (hval & BA_HASH_MASK(a)); #endif
3854832011-11-18Arne Goedeke  BA_PAGE(a, n)->hchain = *b; *b = n; }
6fc6ce2011-12-30Arne Goedeke 
3854832011-11-18Arne Goedeke static inline void ba_htable_replace(const struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  const void * ptr, const ba_page_t n, const ba_page_t new) { ba_page_t hval = hash1(a, ptr); ba_page_t * b = a->htable + (hval & BA_HASH_MASK(a));
3854832011-11-18Arne Goedeke  while (*b) { if (*b == n) { *b = new; BA_PAGE(a, new)->hchain = BA_PAGE(a, n)->hchain;
6fc6ce2011-12-30Arne Goedeke  BA_PAGE(a, n)->hchain = 0;
3854832011-11-18Arne Goedeke  return; } b = &BA_PAGE(a, *b)->hchain; }
6fc6ce2011-12-30Arne Goedeke #ifdef DEBUG fprintf(stderr, "ba_htable_replace(%p, %u, %u)\n", ptr, n, new); fprintf(stderr, "hval: %u, %u, %u\n", hval, hval & BA_HASH_MASK(a), BA_HASH_MASK(a)); ba_print_htable(a); Pike_error("did not find index to replace.\n") #endif
3854832011-11-18Arne Goedeke } static inline void ba_htable_delete(const struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  const void * ptr, const ba_page_t n) { ba_page_t hval = hash1(a, ptr); ba_page_t * b = a->htable + (hval & BA_HASH_MASK(a));
3854832011-11-18Arne Goedeke  while (*b) { if (*b == n) { *b = BA_PAGE(a, n)->hchain;
6fc6ce2011-12-30Arne Goedeke  BA_PAGE(a, n)->hchain = 0;
3854832011-11-18Arne Goedeke  return; } b = &BA_PAGE(a, *b)->hchain; }
6fc6ce2011-12-30Arne Goedeke #ifdef DEBUG ba_print_htable(a); fprintf(stderr, "ba_htable_delete(%p, %u)\n", ptr, n); Pike_error("did not find index to delete.\n") #endif
a5e1b82011-11-09Arne Goedeke }
6fc6ce2011-12-30Arne Goedeke static inline ba_page_t ba_htable_lookup(const struct block_allocator * a, const void * ptr) { int c = 0; ba_page_t n1, n2;
0a3d342011-11-10Arne Goedeke  ba_page p;
6fc6ce2011-12-30Arne Goedeke  n1 = a->htable[hash1(a, ptr) & BA_HASH_MASK(a)]; while (n1) { p = BA_PAGE(a, n1); if (BA_CHECK_PTR(a, p, ptr)) { return n1; } if (c++ > 100) { n2 = a->htable[hash1(a, ptr) & BA_HASH_MASK(a)]; n1 = n2; do { p = BA_PAGE(a, n1); fprintf(stderr, "%d %p -> %d\n", n1, p->data, p->hchain); if (n1 == p->hchain) break; n1 = p->hchain; } while (n2 != n1); Pike_error("hash chain is infinite\n");
67795d2011-11-11Arne Goedeke  }
6fc6ce2011-12-30Arne Goedeke  n1 = p->hchain; } n2 = a->htable[hash2(a, ptr) & BA_HASH_MASK(a)]; while (n2) { p = BA_PAGE(a, n2); if (BA_CHECK_PTR(a, p, ptr)) { return n2; } if (c++ > 100) { Pike_error("hash chain is infinite\n"); } n2 = p->hchain;
67795d2011-11-11Arne Goedeke  }
a5e1b82011-11-09Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  return 0; }
67795d2011-11-11Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke PMOD_EXPORT void ba_check_allocator(struct block_allocator * a, char *fun, char *file, int line) { unsigned int i = 0; int bad = 0;
67795d2011-11-11Arne Goedeke  ba_page p;
6fc6ce2011-12-30Arne Goedeke  if (a->empty_pages > a->num_pages) { fprintf(stderr, "too many empty pages.\n"); bad = 1; } for (i = 1; i <= a->num_pages; i++) { ba_page_t hval; int found = 0; ba_page_t n; p = BA_PAGE(a, i); if (p->blocks_used == a->blocks) { if (p->prev || p->next) { fprintf(stderr, "block is full but in list. next: %u prev: %u", p->next, p->prev); bad = 1;
67795d2011-11-11Arne Goedeke  } }
6fc6ce2011-12-30Arne Goedeke  IF_HASH( hval = hash1(a, BA_LASTBLOCK(a, p)); n = a->htable[hval & BA_HASH_MASK(a)]; while (n) { if (n == i) { found = 1; break; } n = BA_PAGE(a, n)->hchain;
67795d2011-11-11Arne Goedeke  }
6fc6ce2011-12-30Arne Goedeke  if (!found) { fprintf(stderr, "did not find page %d, %p (%u) in htable.\n", i, BA_LASTBLOCK(a, BA_PAGE(a, i)), hval); fprintf(stderr, "looking in bucket %u mask: %d\n", hval & BA_HASH_MASK(a), BA_HASH_MASK(a)); bad = 1; } for (i = 0; i < a->allocated; i++) { n = a->htable[i]; while (n) { p = BA_PAGE(a, n); hval = hash1(a, BA_LASTBLOCK(a, p)); if (i != (hval & BA_HASH_MASK(a))) { fprintf(stderr, "page %u found in wrong bucket %d\n", n, i); bad = 1; } n = p->hchain; } } ); } if (bad) { ba_print_htable(a); fprintf(stderr, "\nCalled from %s:%d:%s\n", fun, line, file); fprintf(stderr, "pages: %u\n", a->num_pages); Pike_error("bad");
a5e1b82011-11-09Arne Goedeke  } }
6fc6ce2011-12-30Arne Goedeke  PMOD_EXPORT void * ba_alloc(struct block_allocator * a) {
e90c492011-11-27Arne Goedeke  //fprintf(stderr, "ba_alloc(%p)\n", a);
a5e1b82011-11-09Arne Goedeke  ba_page p;
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG ba_check_allocator(a, "ba_alloc top", __FILE__, __LINE__); #endif
3854832011-11-18Arne Goedeke #ifndef BA_SEGREGATE
a5e1b82011-11-09Arne Goedeke  size_t i, j;
6fc6ce2011-12-30Arne Goedeke #else int i;
3854832011-11-18Arne Goedeke #endif
a5e1b82011-11-09Arne Goedeke 
3854832011-11-18Arne Goedeke  if (likely(a->first)) { #ifndef BA_SEGREGATE
67795d2011-11-11Arne Goedeke  uintptr_t m;
3854832011-11-18Arne Goedeke #endif
6fc6ce2011-12-30Arne Goedeke  ba_block_header ptr;
a5e1b82011-11-09Arne Goedeke 
d2892b2011-12-25Arne Goedeke #ifdef BA_DEBUG if (a->first > a->num_pages) { fprintf(stderr, "unused page set. %d > num_pages: %d.\n", a->first, a->num_pages); } #endif
3854832011-11-18Arne Goedeke  p = BA_PAGE(a, a->first);
67795d2011-11-11Arne Goedeke 
d2892b2011-12-25Arne Goedeke #ifdef BA_DEBUG if (p->prev) { fprintf(stderr, "a->first has previous: %d\n", p->prev); } #endif
3854832011-11-18Arne Goedeke #ifndef BA_SEGREGATE
67795d2011-11-11Arne Goedeke  i = p->free_mask;
d2892b2011-12-25Arne Goedeke #ifdef BA_DEBUG
3854832011-11-18Arne Goedeke  if (i >= BA_MASK_NUM(a)) { Pike_error("free mask is out of range!\n"); }
d2892b2011-12-25Arne Goedeke #endif
67795d2011-11-11Arne Goedeke  m = p->mask[i];
d2892b2011-12-25Arne Goedeke #ifdef BA_DEBUG
67795d2011-11-11Arne Goedeke  if (!m) { // fprintf(stderr, "blk(%p)# n: %u\tmask: %04X\n", p, i, m);
f25b0b2011-11-10Arne Goedeke  Pike_error("This should not happen!\n"); }
d2892b2011-12-25Arne Goedeke #endif
67795d2011-11-11Arne Goedeke // fprintf(stderr, "blk(%p)> n: %u\tmask: %04X\n", p, i, m); #if SIZEOF_CHAR_P == 8 j = ctz64(m); #else j = ctz32(m); #endif m ^= TBITMASK(uintptr_t, j);
3854832011-11-18Arne Goedeke  //fprintf(stderr, "setting bit %u -> %8X (using %u blocks) maks_num: %u\n", i*sizeof(uintptr_t)*8 + j, m, p->blocks_used, BA_MASK_NUM(a));
67795d2011-11-11Arne Goedeke  p->mask[i] = m; // fprintf(stderr, "blk(%p)< n: %u\tmask: %04X\n", p, i, m);
3854832011-11-18Arne Goedeke  // TODO: remove this: p->blocks_used ++;
e90c492011-11-27Arne Goedeke  if (m) { /* fprintf(stderr, "3 alloced pointer %p (%u/%u used %u)\n", BA_BLOCKN(a, p, (i*sizeof(uintptr_t)*8 + j)), (i*sizeof(uintptr_t)*8 + j), a->blocks, p->blocks_used); */
6fc6ce2011-12-30Arne Goedeke  ptr = BA_BLOCKN(a, p, (i*sizeof(uintptr_t)*8 + j)); goto RETURN_PTR;
e90c492011-11-27Arne Goedeke  } else {
67795d2011-11-11Arne Goedeke  for (m = i+1; m < BA_MASK_NUM(a); m++) { if (p->mask[m]) { p->free_mask = m;
e90c492011-11-27Arne Goedeke  /* fprintf(stderr, "1 alloced pointer %p (%u/%u used %u)\n", BA_BLOCKN(a, p, (i*sizeof(uintptr_t)*8 + j)), (i*sizeof(uintptr_t)*8 + j), a->blocks, p->blocks_used); */
6fc6ce2011-12-30Arne Goedeke  ptr = BA_BLOCKN(a, p, (i*sizeof(uintptr_t)*8 + j)); goto RETURN_PTR;
67795d2011-11-11Arne Goedeke  } }
6fc6ce2011-12-30Arne Goedeke RETURN_PTR:
d2892b2011-12-25Arne Goedeke #ifdef BA_DEBUG
3854832011-11-18Arne Goedeke  if (p->blocks_used != a->blocks) { fprintf(stderr, "wrong block count detected: %u vs %u\n", p->blocks_used, a->blocks); Pike_error("croak\n"); }
d2892b2011-12-25Arne Goedeke #endif
e90c492011-11-27Arne Goedeke  //fprintf(stderr, "page is full now\n");
67795d2011-11-11Arne Goedeke  if (a->last == a->first) { a->first = a->last = 0; } else { a->first = p->next; } p->next = 0; p->free_mask = BA_MASK_NUM(a);
6fc6ce2011-12-30Arne Goedeke  ptr = BA_BLOCKN(a, p, (i*sizeof(uintptr_t)*8 + j));
a5e1b82011-11-09Arne Goedeke  }
3854832011-11-18Arne Goedeke #else
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG if (p->blocks_used == a->blocks) Pike_error("baaad!\n"); #endif
3854832011-11-18Arne Goedeke  p->blocks_used ++;
d2892b2011-12-25Arne Goedeke #ifdef BA_DEBUG if (p->first < 1 || p->first > a->blocks) { fprintf(stderr, "bad index: %d (should be [1..%d]\n", p->first-1, a->blocks); } #endif
3854832011-11-18Arne Goedeke  ptr = BA_BLOCKN(a, p, p->first-1);
d2892b2011-12-25Arne Goedeke  #ifdef BA_DEBUG if (!p->data) { fprintf(stderr, "got null pointer from uninitialized page %d of block %d (num_pages %d).\n", a->first, p->first-1, a->num_pages); } #endif
e90c492011-11-27Arne Goedeke  //fprintf(stderr, "alloced pointer %p (%u/%u used %u)\n", //ptr, p->first-1, a->blocks, p->blocks_used);
3854832011-11-18Arne Goedeke  if (unlikely(p->blocks_used == a->blocks)) { a->first = p->next;
6fc6ce2011-12-30Arne Goedeke  if (a->first)
5df15e2011-12-25Arne Goedeke  BA_PAGE(a, a->first)->prev = 0;
3854832011-11-18Arne Goedeke  p->next = 0; } else {
6fc6ce2011-12-30Arne Goedeke  if (p->blocks_used == 1) a->empty_pages --; p->first = ptr->next; } #endif #ifdef BA_DEBUG if (ptr->magic != BA_MARK_FREE) { fprintf(stderr, "found block with bad magic.\n");
3854832011-11-18Arne Goedeke  }
6fc6ce2011-12-30Arne Goedeke  ptr->magic = BA_MARK_ALLOC; #endif
e90c492011-11-27Arne Goedeke  //fprintf(stderr, "first is %u\n", p->first);
3854832011-11-18Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  return (void*)ptr;
3854832011-11-18Arne Goedeke  }
e90c492011-11-27Arne Goedeke  //fprintf(stderr, "allocating new page. was %p\n", p);
6fc6ce2011-12-30Arne Goedeke  if (unlikely(a->num_pages == a->allocated)) {
3854832011-11-18Arne Goedeke  ba_grow(a);
6fc6ce2011-12-30Arne Goedeke  } #ifdef BA_DEBUG ba_check_allocator(a, "ba_alloc after grow", __FILE__, __LINE__); #endif
3854832011-11-18Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  a->num_pages++; p = BA_PAGE(a, a->num_pages); #ifdef BA_DEBUG
d2892b2011-12-25Arne Goedeke  if (p->data) {
6fc6ce2011-12-30Arne Goedeke  void * new = malloc(BA_PAGESIZE(a)); fprintf(stderr, "reusing unfreed page %d, data: %p -> %p\n", a->num_pages, p->data, new); p->data = new; } else #endif
3854832011-11-18Arne Goedeke  p->data = malloc(BA_PAGESIZE(a));
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG
fdfd282011-12-25Arne Goedeke  _malloc_counter++;
6fc6ce2011-12-30Arne Goedeke #endif
3854832011-11-18Arne Goedeke  if (!p->data) {
6fc6ce2011-12-30Arne Goedeke  Pike_error("no mem. alloc returned zero.");
a5e1b82011-11-09Arne Goedeke  }
3854832011-11-18Arne Goedeke  p->next = p->prev = 0;
6fc6ce2011-12-30Arne Goedeke  a->first = a->num_pages; IF_HASH( ba_htable_insert(a, BA_LASTBLOCK(a, p), a->num_pages); #ifdef BA_DEBUG ba_check_allocator(a, "ba_alloc after insert", __FILE__, __LINE__); #endif );
3854832011-11-18Arne Goedeke #ifndef BA_SEGREGATE
e90c492011-11-27Arne Goedeke  if (BA_MASK_NUM(a) > 1) memset((void*)p->mask, 0xff, (BA_MASK_NUM(a)-1)*sizeof(uintptr_t)); p->mask[BA_MASK_NUM(a)-1] = BA_LAST_MASK(a); p->mask[0] ^= TBITMASK(uintptr_t, 0);
3854832011-11-18Arne Goedeke  p->free_mask = 0; // TODO: remove this: p->blocks_used = 1; #else p->blocks_used = 1; p->first = 2;
6fc6ce2011-12-30Arne Goedeke  for (i = 1; i < a->blocks; i++) { #ifdef BA_DEBUG BA_BLOCKN(a, p, i)->magic = BA_MARK_FREE; #endif BA_BLOCKN(a, p, i)->next = i+2; } //memset(p->data, 0x00, BA_PAGESIZE(a)); #endif #ifdef BA_DEBUG BA_BLOCKN(a, p, 0)->magic = BA_MARK_ALLOC; ba_check_allocator(a, "ba_alloc after insert", __FILE__, __LINE__);
3854832011-11-18Arne Goedeke #endif return p->data;
a5e1b82011-11-09Arne Goedeke }
6fc6ce2011-12-30Arne Goedeke PMOD_EXPORT void ba_free(struct block_allocator * a, void * ptr) {
0a3d342011-11-10Arne Goedeke  uintptr_t t;
3854832011-11-18Arne Goedeke #ifndef BA_SEGREGATE
0a3d342011-11-10Arne Goedeke  unsigned int mask, bit;
3854832011-11-18Arne Goedeke #endif
0a3d342011-11-10Arne Goedeke  ba_page p;
6fc6ce2011-12-30Arne Goedeke  ba_page_t n;
67795d2011-11-11Arne Goedeke  n = a->last_free;
a5e1b82011-11-09Arne Goedeke 
67795d2011-11-11Arne Goedeke  if (n) {
3854832011-11-18Arne Goedeke  p = BA_PAGE(a, n);
67795d2011-11-11Arne Goedeke  if (!BA_CHECK_PTR(a, p, ptr)) n = 0;
a5e1b82011-11-09Arne Goedeke  }
6fc6ce2011-12-30Arne Goedeke 
e90c492011-11-27Arne Goedeke  if (unlikely(!n)) {
6fc6ce2011-12-30Arne Goedeke #ifdef BA_HASH_THLD if (a->allocated > BA_HASH_THLD) { #endif a->last_free = n = ba_htable_lookup(a, ptr); #ifdef BA_HASH_THLD } else { for (t = 1; t <= a->num_pages; t++) { p = BA_PAGE(a, t); if (BA_CHECK_PTR(a, p, ptr)) { a->last_free = n = t; break; } } } #endif
67795d2011-11-11Arne Goedeke 
e90c492011-11-27Arne Goedeke  if (unlikely(!n)) { #ifdef BA_DEBUG fprintf(stderr, "magnitude: %u\n", a->magnitude);
3854832011-11-18Arne Goedeke  fprintf(stderr, "did not find %p (%X[%X] | %X[%X])\n", ptr, hash1(a, ptr), hash1(a, ptr) & BA_HASH_MASK(a), hash2(a, ptr), hash2(a, ptr) & BA_HASH_MASK(a) );
e90c492011-11-27Arne Goedeke  ba_print_htable(a); #endif
67795d2011-11-11Arne Goedeke  Pike_error("Unknown pointer \n"); }
d2892b2011-12-25Arne Goedeke  #ifdef BA_DEBUG if (n > a->num_pages) { fprintf(stderr, "freeing from unknown page %d (num_pages: %d).\n", n, a->num_pages); } #endif
67795d2011-11-11Arne Goedeke 
3854832011-11-18Arne Goedeke  p = BA_PAGE(a, n);
d2892b2011-12-25Arne Goedeke  #ifdef BA_DEBUG if (!BA_CHECK_PTR(a, p, ptr)) { fprintf(stderr, "pointer %p in wrong page %p", ptr, p->data); } #endif
67795d2011-11-11Arne Goedeke  }
0a3d342011-11-10Arne Goedeke  t = (uintptr_t)((char*)ptr - p->data)/a->block_size;
3854832011-11-18Arne Goedeke #ifndef BA_SEGREGATE
0a3d342011-11-10Arne Goedeke  mask = t / (sizeof(uintptr_t) * 8); bit = t & ((sizeof(uintptr_t)*8) - 1);
e90c492011-11-27Arne Goedeke  /* fprintf(stderr, "freeing pointer %p (%u/%u) used %u\n", ptr, t, a->blocks, p->blocks_used); */
a5e1b82011-11-09Arne Goedeke 
67795d2011-11-11Arne Goedeke  t = p->mask[mask]; if (t & TBITMASK(uintptr_t, bit)) {
a5e1b82011-11-09Arne Goedeke  Pike_error("double free!"); }
67795d2011-11-11Arne Goedeke  if (mask < p->free_mask) p->free_mask = mask;
3854832011-11-18Arne Goedeke  // TODO: remove this: p->blocks_used --;
a5e1b82011-11-09Arne Goedeke 
3854832011-11-18Arne Goedeke  if (unlikely(t == 0)) {
e90c492011-11-27Arne Goedeke  unsigned int i;
67795d2011-11-11Arne Goedeke  p->free_mask = mask;
e90c492011-11-27Arne Goedeke  for (i = 0; i < BA_MASK_NUM(a); i++) if (p->mask[i]) { p->mask[mask] = TBITMASK(uintptr_t, bit); return; } p->mask[mask] = TBITMASK(uintptr_t, bit);
3854832011-11-18Arne Goedeke  if (p->blocks_used != a->blocks-1) { fprintf(stderr, "wrong block count detected: %u vs %u\n", p->blocks_used+1, a->blocks); Pike_error("croak\n"); } if (a->first == 0) {
6fc6ce2011-12-30Arne Goedeke  a->first = n;
3854832011-11-18Arne Goedeke  p->next = p->prev = 0; } else { p->next = a->first; BA_PAGE(a, a->first)->prev = n; a->first = n; } return;
a5e1b82011-11-09Arne Goedeke  }
67795d2011-11-11Arne Goedeke  t |= TBITMASK(uintptr_t, bit); p->mask[mask] = t;
a5e1b82011-11-09Arne Goedeke 
3854832011-11-18Arne Goedeke  if (unlikely(~t == 0)) { for (bit = 0; bit < BA_MASK_NUM(a)-1; bit++) if (~p->mask[bit]) return; if (~(p->mask[BA_MASK_NUM(a)-1]|~BA_LAST_MASK(a))) return; if (p->blocks_used != 0) { fprintf(stderr, "wrong block count detected: %u vs %u\n", p->blocks_used+1, 1); Pike_error("croak\n"); } ba_remove_page(a, n);
e90c492011-11-27Arne Goedeke  //fprintf(stderr, "could reclaim memory of page %u\n", n);
3854832011-11-18Arne Goedeke  // we can reclaim memory to the system here! } else if (p->blocks_used == 0 || p->blocks_used == a->blocks - 1) { fprintf(stderr, "wrong number of used blocks: %u\n", p->blocks_used); Pike_error("bad croak\n");
a5e1b82011-11-09Arne Goedeke  }
3854832011-11-18Arne Goedeke #else if (p->blocks_used == a->blocks) {
67795d2011-11-11Arne Goedeke  if (a->first == 0) {
6fc6ce2011-12-30Arne Goedeke  a->first = n;
67795d2011-11-11Arne Goedeke  p->next = p->prev = 0; } else { p->next = a->first; BA_PAGE(a, a->first)->prev = n; a->first = n; }
3854832011-11-18Arne Goedeke  } else if (p->blocks_used == 1) {
fdfd282011-12-25Arne Goedeke  if (a->empty_pages == a->max_empty_pages) { ba_remove_page(a, n);
6fc6ce2011-12-30Arne Goedeke  return; } else a->empty_pages++;
3854832011-11-18Arne Goedeke  } p->blocks_used --;
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG if (((ba_block_header)ptr)->magic == BA_MARK_FREE) { fprintf(stderr, "double freed somethign\n"); } memset(ptr, 0x75, a->block_size); ((ba_block_header)ptr)->magic = BA_MARK_FREE; #endif ((ba_block_header)ptr)->next = p->first;
e90c492011-11-27Arne Goedeke  //fprintf(stderr, "setting first to %u (%p vs %p) n: %u vs %u\n", t+1, BA_BLOCKN(a, p, t+1), ptr, BA_NBLOCK(a, p, BA_BLOCKN(a, p, t+1)), BA_NBLOCK(a, p, ptr));
3854832011-11-18Arne Goedeke  p->first = t+1; #endif } static inline void ba_remove_page(struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  ba_page_t n) {
3854832011-11-18Arne Goedeke  ba_page tmp, p;
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG ba_check_allocator(a, "ba_remove_page", __FILE__, __LINE__); if (a->empty_pages < a->max_empty_pages) { Pike_error("strange things happening\n"); } #endif
3854832011-11-18Arne Goedeke  p = BA_PAGE(a, n);
4750752011-11-19Arne Goedeke  /*
3854832011-11-18Arne Goedeke  fprintf(stderr, "removing page %4u\t(%p .. %p) -> %X (%X) (of %u).\n", n, p->data, BA_LASTBLOCK(a, p), hash1(a, BA_LASTBLOCK(a,p)), hash1(a, BA_LASTBLOCK(a,p)) & BA_HASH_MASK(a), a->num_pages);
4750752011-11-19Arne Goedeke  */
3854832011-11-18Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  IF_HASH( ba_htable_delete(a, BA_LASTBLOCK(a, p), n); );
3854832011-11-18Arne Goedeke  free(p->data); p->data = NULL; if (a->last_free == n) a->last_free = 0; else if (a->last_free == a->num_pages) a->last_free = n; if (p->prev) { tmp = BA_PAGE(a, p->prev); tmp->next = p->next; } else { a->first = p->next; } if (p->next) { tmp = BA_PAGE(a, p->next); tmp->prev = p->prev; }
de2d072011-12-25Arne Goedeke  if (a->num_pages != n) { tmp = BA_PAGE(a, a->num_pages);
6fc6ce2011-12-30Arne Goedeke  // page tmp will have index changed to n IF_HASH( ba_htable_delete(a, BA_LASTBLOCK(a, tmp), a->num_pages); );
de2d072011-12-25Arne Goedeke  /* fprintf(stderr, "replacing with page %u\t(%p .. %p) -> %X (%X)\n", a->num_pages, tmp->data, BA_LASTBLOCK(a, tmp), hash1(a, BA_LASTBLOCK(a,tmp)), hash1(a, BA_LASTBLOCK(a,tmp)) & BA_HASH_MASK(a) ); */ *p = *tmp;
6fc6ce2011-12-30Arne Goedeke  IF_HASH( // ba_htable_replace(a, BA_LASTBLOCK(a, tmp), a->num_pages, n); ba_htable_insert(a, BA_LASTBLOCK(a, tmp), n); );
de2d072011-12-25Arne Goedeke  if (p->next) BA_PAGE(a, p->next)->prev = n; if (p->prev) BA_PAGE(a, p->prev)->next = n;
6fc6ce2011-12-30Arne Goedeke  if (a->num_pages == a->first)
de2d072011-12-25Arne Goedeke  a->first = n; }
6fc6ce2011-12-30Arne Goedeke  memset(BA_PAGE(a, a->num_pages), 0, sizeof(struct ba_page));
de2d072011-12-25Arne Goedeke 
d2892b2011-12-25Arne Goedeke #ifdef BA_DEBUG if (a->first == a->num_pages) { fprintf(stderr, "a->first will be old removed page %d\n", a->first); fprintf(stderr, "page %d was not moved and prev was %d\n", n, p->prev); } #endif
3854832011-11-18Arne Goedeke  a->num_pages--;
6fc6ce2011-12-30Arne Goedeke  #ifdef BA_DEBUG ba_check_allocator(a, "ba_remove_page", __FILE__, __LINE__); #endif if (a->allocated > BA_ALLOC_INITIAL && a->num_pages < (a->allocated >> 2)) { ba_shrink(a); }
a5e1b82011-11-09Arne Goedeke }