0a3d342011-11-10Arne Goedeke #include <stdlib.h>
67795d2011-11-11Arne Goedeke #include <string.h>
3854832011-11-18Arne Goedeke #include <stdint.h>
68bb052012-02-12Arne Goedeke #include <malloc.h>
c2e5e22012-02-08Arne Goedeke #ifdef BA_STATS # include <unistd.h> # include <mcheck.h> #endif
0a3d342011-11-10Arne Goedeke 
a60c242011-12-25Arne Goedeke #include "bitvector.h"
6fc6ce2011-12-30Arne Goedeke 
e2499f2012-01-03Arne Goedeke #if defined(PIKE_CORE) || defined(DYNAMIC_MODULE) || defined(STATIC_MODULE) #include "pike_memory.h" #endif
41889b2012-01-10Arne Goedeke #include "block_allocator.h"
6fc6ce2011-12-30Arne Goedeke 
f2b3bd2011-12-31Arne Goedeke #ifndef PMOD_EXPORT # warning WTF
3854832011-11-18Arne Goedeke #endif
fdfd282011-12-25Arne Goedeke 
689a782012-02-08Arne Goedeke #if (!defined(PIKE_CORE) && !defined(DYNAMIC_MODULE) \
e2499f2012-01-03Arne Goedeke  && !defined(STATIC_MODULES)) PMOD_EXPORT char errbuf[8192];
689a782012-02-08Arne Goedeke #endif
e2499f2012-01-03Arne Goedeke 
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
c8b5902012-01-04Arne Goedeke static INLINE void ba_htable_insert(const struct block_allocator * a,
689a782012-02-08Arne Goedeke  ba_page ptr); static INLINE void ba_htable_grow(struct block_allocator * a);
68bb052012-02-12Arne Goedeke #endif
6fc6ce2011-12-30Arne Goedeke 
335e6a2012-01-04Arne Goedeke #define BA_NBLOCK(a, p, ptr) ((uintptr_t)((char*)ptr - (char)(p+1))/(a)->block_size)
e90c492011-11-27Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke #define BA_DIVIDE(a, b) ((a) / (b) + (!!((a) & ((b)-1))))
99901f2012-01-28Arne Goedeke #define BA_PAGESIZE(a) (sizeof(struct ba_page) + (a)->blocks * (a)->block_size)
3854832011-11-18Arne Goedeke #define BA_HASH_MASK(a) (((a->allocated)) - 1)
e2499f2012-01-03Arne Goedeke #ifdef BA_HASH_THLD
335e6a2012-01-04Arne Goedeke # define BA_BYTES(a) ( (sizeof(ba_page) + ((a->allocated > BA_HASH_THLD) ? sizeof(ba_page_t) : 0)) * ((a)->allocated) )
e2499f2012-01-03Arne Goedeke #else
689a782012-02-08Arne Goedeke # define BA_BYTES(a) ( (sizeof(ba_page) * ((a)->allocated) )
6fc6ce2011-12-30Arne Goedeke #endif
a5e1b82011-11-09Arne Goedeke 
41889b2012-01-10Arne Goedeke #define PRINT_NODE(a, name) do {\
689a782012-02-08Arne Goedeke  fprintf(stderr, #name": %p\n", a->name);\
41889b2012-01-10Arne Goedeke } while (0)
689a782012-02-08Arne Goedeke PMOD_EXPORT void ba_show_pages(const struct block_allocator * a) {
e2499f2012-01-03Arne Goedeke  unsigned int i = 0;
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
41889b2012-01-10Arne Goedeke  fprintf(stderr, "allocated: %u\n", a->allocated);
68bb052012-02-12Arne Goedeke #endif
41889b2012-01-10Arne Goedeke  fprintf(stderr, "num_pages: %u\n", a->num_pages); fprintf(stderr, "max_empty_pages: %u\n", a->max_empty_pages); fprintf(stderr, "empty_pages: %u\n", a->empty_pages); fprintf(stderr, "magnitude: %u\n", a->magnitude); PRINT_NODE(a, empty); PRINT_NODE(a, first);
68bb052012-02-12Arne Goedeke #ifdef BA_USE_MEMALIGN PRINT_NODE(a, full); #else
41889b2012-01-10Arne Goedeke  PRINT_NODE(a, last_free);
68bb052012-02-12Arne Goedeke #endif
41889b2012-01-10Arne Goedeke 
689a782012-02-08Arne Goedeke  PAGE_LOOP(a, {
41889b2012-01-10Arne Goedeke  ba_block_t blocks_used;
dfad0b2012-02-12Arne Goedeke  blocks_used = p->used;
689a782012-02-08Arne Goedeke  fprintf(stderr, "(%p)\t%f\t(%u %d) --> (prev: %p | next: %p)\n", p, blocks_used/(double)a->blocks * 100,
41889b2012-01-10Arne Goedeke  blocks_used, blocks_used,
689a782012-02-08Arne Goedeke  p->prev, p->next); });
e2499f2012-01-03Arne Goedeke }
17cb1e2012-02-08Arne Goedeke #ifdef BA_STATS
fc30092012-02-08Arne Goedeke # define PRCOUNT(X) fprintf(stat_file, #X " %5.1lf %% ", (a->stats.X/all)*100); static FILE * stat_file = NULL; ATTRIBUTE((constructor)) static void ba_stats_init() { char * fname = getenv("BA_STAT_TRACE"); if (fname) { stat_file = fopen(fname, "w"); } else stat_file = stderr; } ATTRIBUTE((destructor)) static void ba_stats_deinit() { if (stat_file != stderr) { fclose(stat_file); stat_file = stderr; } }
17cb1e2012-02-08Arne Goedeke PMOD_EXPORT void ba_print_stats(struct block_allocator * a) { FILE *f;
fc30092012-02-08Arne Goedeke  int i; double all;
d446a62012-02-08Arne Goedeke  struct block_alloc_stats * s = &a->stats; size_t used = s->st_max * a->block_size; size_t malloced = s->st_max_pages * (a->block_size * a->blocks); size_t overhead = s->st_max_pages * sizeof(struct ba_page); int mall = s->st_mallinfo.uordblks; int moverhead = s->st_mallinfo.fordblks;
fc30092012-02-08Arne Goedeke  const char * fmt = "%s:\n max used %.1lf kb in %.1lf kb (#%lld) pages"
17cb1e2012-02-08Arne Goedeke  " (overhead %.2lf kb)"
d446a62012-02-08Arne Goedeke  " mall: %.1lf kb overhead: %.1lf kb "
17cb1e2012-02-08Arne Goedeke  " page_size: %d block_size: %d\n";
5654182012-02-08Arne Goedeke  overhead += s->st_max_allocated * 2 * sizeof(void*);
d446a62012-02-08Arne Goedeke  if (s->st_max == 0) return;
fc30092012-02-08Arne Goedeke  fprintf(stat_file, fmt,
d446a62012-02-08Arne Goedeke  s->st_name,
17cb1e2012-02-08Arne Goedeke  used / 1024.0, malloced / 1024.0,
d446a62012-02-08Arne Goedeke  s->st_max_pages, overhead / 1024.0,
17cb1e2012-02-08Arne Goedeke  mall / 1024.0, moverhead / 1024.0, a->block_size * a->blocks, a->block_size );
fc30092012-02-08Arne Goedeke  all = ((a->stats.free_fast1 + a->stats.free_fast2 + a->stats.find_linear + a->stats.find_hash + a->stats.free_empty + a->stats.free_full)); if (all == 0.0) return; fprintf(stat_file, "free COUNTS: ");
5654182012-02-08Arne Goedeke  PRCOUNT(free_fast1); PRCOUNT(free_fast2); PRCOUNT(find_linear); PRCOUNT(find_hash); PRCOUNT(free_empty); PRCOUNT(free_full);
fc30092012-02-08Arne Goedeke  fprintf(stat_file, " all %lu\n", (long unsigned int)all);
17cb1e2012-02-08Arne Goedeke } #endif
d947bc2012-01-06Arne Goedeke //#define BA_ALIGNMENT 8
c8b5902012-01-04Arne Goedeke PMOD_EXPORT INLINE void ba_init(struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  uint32_t block_size, ba_page_t blocks) {
d947bc2012-01-06Arne Goedeke  uint32_t page_size; #ifdef BA_ALIGNMENT if (block_size & (BA_ALIGNMENT - 1)) block_size += (BA_ALIGNMENT - (block_size & (BA_ALIGNMENT - 1))); #endif
99901f2012-01-28Arne Goedeke  page_size = block_size * blocks + BLOCK_HEADER_SIZE;
3854832011-11-18Arne Goedeke 
68bb052012-02-12Arne Goedeke  a->empty = a->first = NULL; #ifdef BA_USE_MEMALIGN a->full = NULL; #else a->last_free = NULL; #endif
a5e1b82011-11-09Arne Goedeke 
99901f2012-01-28Arne Goedeke #ifdef BA_DEBUG fprintf(stderr, "blocks: %u block_size: %u page_size: %u\n", blocks, block_size, page_size); #endif // is not multiple of memory page size
25a9422012-01-06Arne Goedeke  if ((page_size & (page_size - 1))) { page_size = round_up32(page_size);
99901f2012-01-28Arne Goedeke  } a->blocks = (page_size - BLOCK_HEADER_SIZE)/block_size; a->offset = sizeof(struct ba_page) + (a->blocks - 1) * block_size; #ifdef BA_DEBUG
689a782012-02-08Arne Goedeke  fprintf(stderr, "blocks: %u block_size: %u page_size: %u mallocing size: %lu, next pages: %u\n",
99901f2012-01-28Arne Goedeke  a->blocks, block_size, blocks * block_size, BA_PAGESIZE(a)+sizeof(struct ba_page), round_up32(BA_PAGESIZE(a)+sizeof(struct ba_page))); #endif
a5e1b82011-11-09Arne Goedeke 
25a9422012-01-06Arne Goedeke  a->magnitude = (uint16_t)ctz32(page_size);
0a3d342011-11-10Arne Goedeke  a->block_size = block_size;
a5e1b82011-11-09Arne Goedeke  a->num_pages = 0;
9387262011-12-25Arne Goedeke  a->empty_pages = 0;
e2499f2012-01-03Arne Goedeke  a->max_empty_pages = BA_MAX_EMPTY;
a5e1b82011-11-09Arne Goedeke 
67795d2011-11-11Arne Goedeke  // we start with management structures for 16 pages
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
6fc6ce2011-12-30Arne Goedeke  a->allocated = BA_ALLOC_INITIAL;
689a782012-02-08Arne Goedeke  a->pages = (ba_page*)malloc(BA_ALLOC_INITIAL * sizeof(ba_page));
5654182012-02-08Arne Goedeke  if (!a->pages) Pike_error("nomem");
689a782012-02-08Arne Goedeke  memset(a->pages, 0, BA_ALLOC_INITIAL * sizeof(ba_page));
68bb052012-02-12Arne Goedeke #endif
67795d2011-11-11Arne Goedeke }
2861c22012-01-28Arne Goedeke static INLINE void ba_free_page(struct block_allocator * a, ba_page p) { p->first = BA_BLOCKN(a, p, 0); p->used = 0; if (a->blueprint) { //xmemset(p+1, a->blueprint, a->block_size, a->blocks); size_t len = (a->blocks - 1) * a->block_size, clen = a->block_size; p++; memcpy(p, a->blueprint, a->block_size); while (len > clen) { memcpy(((char*)(p)) + clen, p, clen); len -= clen; clen <<= 1; } if (len) memcpy(((char*)(p)) + clen, p, len); p--; }
d4cc302012-01-28Arne Goedeke #ifdef BA_CHAIN_PAGE
2861c22012-01-28Arne Goedeke  do { char * ptr = (char*)(p+1); while (ptr < BA_LASTBLOCK(a, p)) { #ifdef BA_DEBUG PIKE_MEM_RW(((ba_block_header)ptr)->magic); ((ba_block_header)ptr)->magic = BA_MARK_FREE; #endif ((ba_block_header)ptr)->next = (ba_block_header)(ptr+a->block_size); ptr+=a->block_size; } BA_LASTBLOCK(a, p)->next = NULL; } while (0);
d4cc302012-01-28Arne Goedeke #else p->first->next = BA_ONE; #endif
2861c22012-01-28Arne Goedeke }
c8b5902012-01-04Arne Goedeke PMOD_EXPORT INLINE void ba_free_all(struct block_allocator * a) {
4750752011-11-19Arne Goedeke  unsigned int i;
67795d2011-11-11Arne Goedeke 
68bb052012-02-12Arne Goedeke #ifdef BA_USE_MEMALIGN if (!a->num_pages) return; #else
e2499f2012-01-03Arne Goedeke  if (!a->allocated) return;
68bb052012-02-12Arne Goedeke #endif
e2499f2012-01-03Arne Goedeke 
689a782012-02-08Arne Goedeke  PAGE_LOOP(a, { PIKE_MEM_RW_RANGE(p, BA_PAGESIZE(a));
335e6a2012-01-04Arne Goedeke  free(p);
689a782012-02-08Arne Goedeke  });
67795d2011-11-11Arne Goedeke  a->num_pages = 0;
6fc6ce2011-12-30Arne Goedeke  a->empty_pages = 0;
68bb052012-02-12Arne Goedeke  a->empty = a->first = NULL; #ifdef BA_USE_MEMALIGN a->full = NULL; #else a->last_free = NULL;
e2499f2012-01-03Arne Goedeke  a->allocated = BA_ALLOC_INITIAL;
689a782012-02-08Arne Goedeke  memset(a->pages, 0, a->allocated * sizeof(ba_page));
68bb052012-02-12Arne Goedeke #endif
67795d2011-11-11Arne Goedeke }
c8b5902012-01-04Arne Goedeke PMOD_EXPORT INLINE void ba_count_all(struct block_allocator * a, size_t *num, size_t *size) {
4750752011-11-19Arne Goedeke  unsigned int i;
3854832011-11-18Arne Goedeke  size_t n = 0;
f2b3bd2011-12-31Arne Goedeke  //fprintf(stderr, "page_size: %u, pages: %u\n", BA_PAGESIZE(a), a->num_pages);
68bb052012-02-12Arne Goedeke #ifdef BA_USE_MEMALIGN *size = a->num_pages * BA_PAGESIZE(a); #else
3854832011-11-18Arne Goedeke  *size = BA_BYTES(a) + a->num_pages * BA_PAGESIZE(a);
68bb052012-02-12Arne Goedeke #endif
689a782012-02-08Arne Goedeke  PAGE_LOOP(a, { n += p->used; });
3854832011-11-18Arne Goedeke  *num = n; }
c8b5902012-01-04Arne Goedeke PMOD_EXPORT INLINE void ba_destroy(struct block_allocator * a) {
281be72012-01-28Arne Goedeke  unsigned int i;
689a782012-02-08Arne Goedeke  PAGE_LOOP(a, { PIKE_MEM_RW_RANGE(p, BA_PAGESIZE(a));
281be72012-01-28Arne Goedeke  free(p);
689a782012-02-08Arne Goedeke  });
281be72012-01-28Arne Goedeke 
68bb052012-02-12Arne Goedeke #ifdef BA_USE_MEMALIGN a->full = NULL; #else
f2b3bd2011-12-31Arne Goedeke  PIKE_MEM_RW_RANGE(a->pages, BA_BYTES(a));
67795d2011-11-11Arne Goedeke  free(a->pages);
281be72012-01-28Arne Goedeke  a->last_free = NULL;
68bb052012-02-12Arne Goedeke  a->allocated = 0;
67795d2011-11-11Arne Goedeke  a->pages = NULL;
68bb052012-02-12Arne Goedeke #endif a->empty = a->first = NULL;
281be72012-01-28Arne Goedeke  a->empty_pages = 0; a->num_pages = 0;
17cb1e2012-02-08Arne Goedeke #ifdef BA_STATS
d446a62012-02-08Arne Goedeke  a->stats.st_max = a->stats.st_used = 0;
17cb1e2012-02-08Arne Goedeke #endif
a5e1b82011-11-09Arne Goedeke }
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
c8b5902012-01-04Arne Goedeke static INLINE void ba_grow(struct block_allocator * a) {
3854832011-11-18Arne Goedeke  if (a->allocated) {
6fc6ce2011-12-30Arne Goedeke  // try to detect 32bit overrun? if (a->allocated >= ((ba_page_t)1 << (sizeof(ba_page_t)*8-1))) {
41889b2012-01-10Arne Goedeke  BA_ERROR("too many pages.\n");
6fc6ce2011-12-30Arne Goedeke  }
689a782012-02-08Arne Goedeke  ba_htable_grow(a);
3854832011-11-18Arne Goedeke  } else { ba_init(a, a->block_size, a->blocks); }
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG ba_check_allocator(a, "ba_grow", __FILE__, __LINE__); #endif
67795d2011-11-11Arne Goedeke }
6fc6ce2011-12-30Arne Goedeke #define MIX(t) do { \ t ^= (t >> 20) ^ (t >> 12); \ t ^= (t >> 7) ^ (t >> 4); \ } while(0)
a5e1b82011-11-09Arne Goedeke 
c8b5902012-01-04Arne Goedeke static INLINE ba_page_t hash1(const struct block_allocator * a,
67795d2011-11-11Arne Goedeke  const void * ptr) { uintptr_t t = ((uintptr_t)ptr) >> a->magnitude;
a5e1b82011-11-09Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  MIX(t); return (ba_page_t) t;
a5e1b82011-11-09Arne Goedeke }
c8b5902012-01-04Arne Goedeke static INLINE ba_page_t hash2(const struct block_allocator * a,
67795d2011-11-11Arne Goedeke  const void * ptr) { uintptr_t t = ((uintptr_t)ptr) >> a->magnitude;
a5e1b82011-11-09Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  t++; MIX(t); return (ba_page_t) t; }
689a782012-02-08Arne Goedeke static INLINE void ba_htable_grow(struct block_allocator * a) { ba_page_t n, old; old = a->allocated; a->allocated *= 2; a->pages = (ba_page*)realloc(a->pages, a->allocated * sizeof(ba_page));
5654182012-02-08Arne Goedeke  if (!a->pages) Pike_error("nomem");
689a782012-02-08Arne Goedeke  memset(a->pages+old, 0, old * sizeof(ba_page)); for (n = 0; n < old; n++) { ba_page * b = a->pages + n; while (*b) { ba_page_t h = hash1(a, BA_LASTBLOCK(a, *b)) & BA_HASH_MASK(a); if (h != n) { ba_page p = *b; *b = p->hchain; p->hchain = a->pages[h]; a->pages[h] = p; continue; } b = &(*b)->hchain; } } } static INLINE void ba_htable_shrink(struct block_allocator * a) { ba_page_t n; a->allocated /= 2; for (n = 0; n < a->allocated; n++) { ba_page p = a->pages[a->allocated + n]; if (p) { ba_page t = p; if (a->pages[n]) { while (t->hchain) t = t->hchain; t->hchain = a->pages[n]; } a->pages[n] = p; } } a->pages = (ba_page*)realloc(a->pages, a->allocated * sizeof(ba_page));
5654182012-02-08Arne Goedeke  if (!a->pages) Pike_error("nomem");
689a782012-02-08Arne Goedeke }
f2b3bd2011-12-31Arne Goedeke #ifdef BA_DEBUG PMOD_EXPORT void ba_print_htable(const struct block_allocator * a) {
6fc6ce2011-12-30Arne Goedeke  unsigned int i;
689a782012-02-08Arne Goedeke  for (i = 0; i < a->allocated; i++) { ba_page p = a->pages[i]; ba_page_t hval; if (!p) { fprintf(stderr, "empty %u\n", i);
6fc6ce2011-12-30Arne Goedeke  }
689a782012-02-08Arne Goedeke  while (p) { void * ptr = BA_LASTBLOCK(a, p); hval = hash1(a, ptr); fprintf(stderr, "%u\t%X[%u]\t%p-%p\t%X\n", i, hval, hval & BA_HASH_MASK(a), p+1, ptr, (unsigned int)((uintptr_t)ptr >> a->magnitude)); p = p->hchain; } }
a5e1b82011-11-09Arne Goedeke }
f2b3bd2011-12-31Arne Goedeke #endif
a5e1b82011-11-09Arne Goedeke  /* * insert the pointer to an allocated page into the * hashtable. uses linear probing and open allocation. */
c8b5902012-01-04Arne Goedeke static INLINE void ba_htable_insert(const struct block_allocator * a,
689a782012-02-08Arne Goedeke  ba_page p) { ba_page_t hval = hash1(a, BA_LASTBLOCK(a, p)); ba_page * b = a->pages + (hval & BA_HASH_MASK(a));
6fc6ce2011-12-30Arne Goedeke  #ifdef BA_DEBUG while (*b) {
689a782012-02-08Arne Goedeke  if (*b == p) { fprintf(stderr, "inserting (%p) twice\n", p); fprintf(stderr, "is (%p)\n", *b);
41889b2012-01-10Arne Goedeke  BA_ERROR("double insert.\n");
6fc6ce2011-12-30Arne Goedeke  return; }
689a782012-02-08Arne Goedeke  b = &(*b)->hchain;
6fc6ce2011-12-30Arne Goedeke  }
689a782012-02-08Arne Goedeke  b = a->pages + (hval & BA_HASH_MASK(a));
6fc6ce2011-12-30Arne Goedeke #endif
3854832011-11-18Arne Goedeke 
41889b2012-01-10Arne Goedeke #if 0//def BA_DEBUG
e2499f2012-01-03Arne Goedeke  fprintf(stderr, "replacing bucket %u with page %u by %u\n", hval & BA_HASH_MASK(a), *b, n); #endif
689a782012-02-08Arne Goedeke  p->hchain = *b; *b = p;
3854832011-11-18Arne Goedeke }
6fc6ce2011-12-30Arne Goedeke 
689a782012-02-08Arne Goedeke #if 0
c8b5902012-01-04Arne Goedeke static INLINE void ba_htable_replace(const struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  const void * ptr, const ba_page_t n, const ba_page_t new) { ba_page_t hval = hash1(a, ptr); ba_page_t * b = a->htable + (hval & BA_HASH_MASK(a));
3854832011-11-18Arne Goedeke  while (*b) { if (*b == n) { *b = new; BA_PAGE(a, new)->hchain = BA_PAGE(a, n)->hchain;
6fc6ce2011-12-30Arne Goedeke  BA_PAGE(a, n)->hchain = 0;
3854832011-11-18Arne Goedeke  return; } b = &BA_PAGE(a, *b)->hchain; }
6fc6ce2011-12-30Arne Goedeke #ifdef DEBUG fprintf(stderr, "ba_htable_replace(%p, %u, %u)\n", ptr, n, new); fprintf(stderr, "hval: %u, %u, %u\n", hval, hval & BA_HASH_MASK(a), BA_HASH_MASK(a)); ba_print_htable(a);
41889b2012-01-10Arne Goedeke  BA_ERROR("did not find index to replace.\n")
6fc6ce2011-12-30Arne Goedeke #endif
3854832011-11-18Arne Goedeke }
689a782012-02-08Arne Goedeke #endif
3854832011-11-18Arne Goedeke 
c8b5902012-01-04Arne Goedeke static INLINE void ba_htable_delete(const struct block_allocator * a,
689a782012-02-08Arne Goedeke  ba_page p) { ba_page_t hval = hash1(a, BA_LASTBLOCK(a, p)); ba_page * b = a->pages + (hval & BA_HASH_MASK(a));
3854832011-11-18Arne Goedeke  while (*b) {
689a782012-02-08Arne Goedeke  if (*b == p) { *b = p->hchain; p->hchain = 0;
3854832011-11-18Arne Goedeke  return; }
689a782012-02-08Arne Goedeke  b = &(*b)->hchain;
3854832011-11-18Arne Goedeke  }
6fc6ce2011-12-30Arne Goedeke #ifdef DEBUG
e2499f2012-01-03Arne Goedeke  ba_print_htable(a);
689a782012-02-08Arne Goedeke  fprintf(stderr, "ba_htable_delete(%p, %u)\n", p, n);
41889b2012-01-10Arne Goedeke  BA_ERROR("did not find index to delete.\n")
6fc6ce2011-12-30Arne Goedeke #endif
a5e1b82011-11-09Arne Goedeke }
1939242012-02-08Arne Goedeke static INLINE void ba_htable_lookup(struct block_allocator * a, const void * ptr) {
2d67de2012-02-12Arne Goedeke  ba_page p; ba_page_t h[2]; unsigned char c, b = 0; h[0] = hash1(a, ptr); h[1] = hash2(a, ptr); c = ((uintptr_t)ptr >> (a->magnitude - 1)) & 1; LOOKUP: p = a->pages[h[c] & BA_HASH_MASK(a)]; while (p) { if (BA_CHECK_PTR(a, p, ptr)) { a->last_free = p; return;
6fc6ce2011-12-30Arne Goedeke  }
2d67de2012-02-12Arne Goedeke  p = p->hchain;
67795d2011-11-11Arne Goedeke  }
2d67de2012-02-12Arne Goedeke  c = !c; if (!(b++)) goto LOOKUP;
6fc6ce2011-12-30Arne Goedeke }
68bb052012-02-12Arne Goedeke #endif
67795d2011-11-11Arne Goedeke 
f2b3bd2011-12-31Arne Goedeke #ifdef BA_DEBUG
c8b5902012-01-04Arne Goedeke PMOD_EXPORT INLINE void ba_check_allocator(struct block_allocator * a,
6fc6ce2011-12-30Arne Goedeke  char *fun, char *file, int line) {
689a782012-02-08Arne Goedeke  ba_page_t n;
6fc6ce2011-12-30Arne Goedeke  int bad = 0;
67795d2011-11-11Arne Goedeke  ba_page p;
6fc6ce2011-12-30Arne Goedeke  if (a->empty_pages > a->num_pages) { fprintf(stderr, "too many empty pages.\n"); bad = 1; }
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
689a782012-02-08Arne Goedeke  for (n = 0; n < a->allocated; n++) {
6fc6ce2011-12-30Arne Goedeke  ba_page_t hval; int found = 0;
689a782012-02-08Arne Goedeke  p = a->pages[n]; while (p) { if (!p->first && p != a->first) { if (p->prev || p->next) { fprintf(stderr, "page %p is full but in list. next: %p" "prev: %p\n", p, p->next, p->prev); bad = 1;
6fc6ce2011-12-30Arne Goedeke  }
67795d2011-11-11Arne Goedeke  }
6fc6ce2011-12-30Arne Goedeke 
689a782012-02-08Arne Goedeke  hval = hash1(a, BA_LASTBLOCK(a, p)) & BA_HASH_MASK(a); if (hval != n) { fprintf(stderr, "page with hash %u found in bucket %u\n", hval, n);
6fc6ce2011-12-30Arne Goedeke  bad = 1; }
689a782012-02-08Arne Goedeke  p = p->hchain; }
6fc6ce2011-12-30Arne Goedeke 
689a782012-02-08Arne Goedeke  if (bad) ba_print_htable(a);
6fc6ce2011-12-30Arne Goedeke  }
68bb052012-02-12Arne Goedeke #endif
6fc6ce2011-12-30Arne Goedeke  if (bad) {
41889b2012-01-10Arne Goedeke  ba_show_pages(a);
6fc6ce2011-12-30Arne Goedeke  fprintf(stderr, "\nCalled from %s:%d:%s\n", fun, line, file); fprintf(stderr, "pages: %u\n", a->num_pages);
41889b2012-01-10Arne Goedeke  BA_ERROR("bad");
a5e1b82011-11-09Arne Goedeke  } }
f2b3bd2011-12-31Arne Goedeke #endif
a5e1b82011-11-09Arne Goedeke 
41889b2012-01-10Arne Goedeke static INLINE void ba_alloc_page(struct block_allocator * a) {
a5e1b82011-11-09Arne Goedeke  ba_page p;
e2499f2012-01-03Arne Goedeke  unsigned int i;
d2892b2011-12-25Arne Goedeke 
68bb052012-02-12Arne Goedeke #ifdef BA_USE_MEMALIGN if (a->num_pages == 0) ba_init(a, a->block_size, a->blocks); #else
6fc6ce2011-12-30Arne Goedeke  if (unlikely(a->num_pages == a->allocated)) {
3854832011-11-18Arne Goedeke  ba_grow(a);
6fc6ce2011-12-30Arne Goedeke  }
68bb052012-02-12Arne Goedeke #endif
3854832011-11-18Arne Goedeke 
6fc6ce2011-12-30Arne Goedeke  a->num_pages++;
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
689a782012-02-08Arne Goedeke  p = (ba_page)malloc(BA_PAGESIZE(a));
68bb052012-02-12Arne Goedeke #else p = (ba_page)memalign((size_t)(1 << a->magnitude), BA_PAGESIZE(a)); #endif
689a782012-02-08Arne Goedeke  if (!p) BA_ERROR("no mem. alloc returned zero.");
2861c22012-01-28Arne Goedeke  ba_free_page(a, p);
335e6a2012-01-04Arne Goedeke  p->next = p->prev = NULL;
dfad0b2012-02-12Arne Goedeke  a->first = p;
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
689a782012-02-08Arne Goedeke  ba_htable_insert(a, p);
68bb052012-02-12Arne Goedeke #endif
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG
689a782012-02-08Arne Goedeke  ba_check_allocator(a, "ba_alloc after insert", __FILE__, __LINE__);
6fc6ce2011-12-30Arne Goedeke #endif #ifdef BA_DEBUG
e2499f2012-01-03Arne Goedeke  PIKE_MEM_RW(BA_LASTBLOCK(a, p)->magic);
f2b3bd2011-12-31Arne Goedeke  BA_LASTBLOCK(a, p)->magic = BA_MARK_FREE;
e2499f2012-01-03Arne Goedeke  PIKE_MEM_RW(BA_BLOCKN(a, p, 0)->magic);
6fc6ce2011-12-30Arne Goedeke  BA_BLOCKN(a, p, 0)->magic = BA_MARK_ALLOC; ba_check_allocator(a, "ba_alloc after insert", __FILE__, __LINE__);
3854832011-11-18Arne Goedeke #endif
a5e1b82011-11-09Arne Goedeke }
41889b2012-01-10Arne Goedeke PMOD_EXPORT void ba_low_alloc(struct block_allocator * a) { //fprintf(stderr, "ba_alloc(%p)\n", a);
e2499f2012-01-03Arne Goedeke #ifdef BA_DEBUG
41889b2012-01-10Arne Goedeke  ba_check_allocator(a, "ba_alloc top", __FILE__, __LINE__);
1133892012-01-06Arne Goedeke #endif
41889b2012-01-10Arne Goedeke  //fprintf(stderr, "allocating new page. was %p\n", p); if (a->first) { ba_page p = a->first;
dfad0b2012-02-12Arne Goedeke  DOUBLE_SHIFT(a->first);
68bb052012-02-12Arne Goedeke #ifdef BA_USE_MEMALIGN DOUBLE_LINK(a->full, p); #else
41889b2012-01-10Arne Goedeke  p->next = NULL;
68bb052012-02-12Arne Goedeke #endif
bdead72012-02-12Arne Goedeke  p->first = NULL;
e2499f2012-01-03Arne Goedeke  }
dfad0b2012-02-12Arne Goedeke  if (a->first) return; if (a->empty_pages) { a->first = a->empty; SINGLE_SHIFT(a->empty);
e2499f2012-01-03Arne Goedeke  a->empty_pages--;
41889b2012-01-10Arne Goedeke  a->first->next = NULL; } else ba_alloc_page(a);
2861c22012-01-28Arne Goedeke #ifdef BA_DEBUG
41889b2012-01-10Arne Goedeke  if (!a->first->first) { ba_show_pages(a); BA_ERROR("a->first has no first block!\n"); } #endif #ifdef BA_DEBUG ba_check_allocator(a, "ba_alloc after grow", __FILE__, __LINE__); #endif }
2861c22012-01-28Arne Goedeke PMOD_EXPORT void ba_low_free(struct block_allocator * a, ba_page p, ba_block_header ptr) { // page was full if (unlikely(!ptr->next)) {
5654182012-02-08Arne Goedeke  INC(free_full);
68bb052012-02-12Arne Goedeke #ifdef BA_USE_MEMALIGN DOUBLE_UNLINK(a->full, p); #endif
bdead72012-02-12Arne Goedeke #if 1 if (a->first) { p->prev = a->first; p->next = a->first->next; a->first->next = p; if (p->next) p->next->prev = p; } else #endif DOUBLE_LINK(a->first, p);
2861c22012-01-28Arne Goedeke  } else if (!p->used) {
5654182012-02-08Arne Goedeke  INC(free_empty);
2861c22012-01-28Arne Goedeke  if (a->empty_pages == a->max_empty_pages) { ba_remove_page(a, p); return; }
dfad0b2012-02-12Arne Goedeke  DOUBLE_UNLINK(a->first, p); SINGLE_LINK(a->empty, p);
2861c22012-01-28Arne Goedeke  a->empty_pages ++;
41889b2012-01-10Arne Goedeke  } }
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
1939242012-02-08Arne Goedeke static INLINE void ba_htable_linear_lookup(struct block_allocator * a, const void * ptr) { PAGE_LOOP(a, { if (BA_CHECK_PTR(a, p, ptr)) { a->last_free = p; return; } }); }
bc93792012-02-02Arne Goedeke PMOD_EXPORT INLINE void ba_find_page(struct block_allocator * a, const void * ptr) {
1939242012-02-08Arne Goedeke  a->last_free = NULL;
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG
41889b2012-01-10Arne Goedeke  ba_check_allocator(a, "ba_low_free", __FILE__, __LINE__); #endif #ifdef BA_HASH_THLD if (a->num_pages <= BA_HASH_THLD) {
5654182012-02-08Arne Goedeke  INC(find_linear);
1939242012-02-08Arne Goedeke  ba_htable_linear_lookup(a, ptr);
41889b2012-01-10Arne Goedeke  } else { #endif
5654182012-02-08Arne Goedeke  INC(find_hash);
1939242012-02-08Arne Goedeke  ba_htable_lookup(a, ptr);
41889b2012-01-10Arne Goedeke #ifdef BA_HASH_THLD
6fc6ce2011-12-30Arne Goedeke  }
1939242012-02-08Arne Goedeke  if (a->last_free) return;
3854832011-11-18Arne Goedeke #endif
e2499f2012-01-03Arne Goedeke #ifdef BA_DEBUG
41889b2012-01-10Arne Goedeke  fprintf(stderr, "magnitude: %u\n", a->magnitude); fprintf(stderr, "allocated: %u\n", a->allocated); fprintf(stderr, "did not find %p (%X[%X] | %X[%X])\n", ptr, hash1(a, ptr), hash1(a, ptr) & BA_HASH_MASK(a), hash2(a, ptr), hash2(a, ptr) & BA_HASH_MASK(a) ); ba_print_htable(a);
e2499f2012-01-03Arne Goedeke #endif
4b2d702012-02-08Arne Goedeke  BA_ERROR("Unknown pointer (not found in hash) %lx\n", (long int)ptr);
3854832011-11-18Arne Goedeke }
68bb052012-02-12Arne Goedeke #endif
f2b3bd2011-12-31Arne Goedeke 
41889b2012-01-10Arne Goedeke PMOD_EXPORT void ba_remove_page(struct block_allocator * a, ba_page p) {
6fc6ce2011-12-30Arne Goedeke #ifdef BA_DEBUG ba_check_allocator(a, "ba_remove_page", __FILE__, __LINE__); if (a->empty_pages < a->max_empty_pages) {
41889b2012-01-10Arne Goedeke  BA_ERROR("strange things happening\n"); } if (p == a->first) { BA_ERROR("removing first page. this is not supposed to happen\n");
6fc6ce2011-12-30Arne Goedeke  } #endif
3854832011-11-18Arne Goedeke 
41889b2012-01-10Arne Goedeke #if 0//def BA_DEBUG
e2499f2012-01-03Arne Goedeke  fprintf(stderr, "> ===============\n"); ba_show_pages(a); ba_print_htable(a);
41889b2012-01-10Arne Goedeke  fprintf(stderr, "removing page %4u[%p]\t(%p .. %p) -> %X (%X) (of %u).\n", n, p, p+1, BA_LASTBLOCK(a, p), hash1(a, BA_LASTBLOCK(a,p)),
3854832011-11-18Arne Goedeke  hash1(a, BA_LASTBLOCK(a,p)) & BA_HASH_MASK(a), a->num_pages);
e2499f2012-01-03Arne Goedeke #endif
3854832011-11-18Arne Goedeke 
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
689a782012-02-08Arne Goedeke  ba_htable_delete(a, p);
335e6a2012-01-04Arne Goedeke  PIKE_MEM_RW_RANGE(*p, BA_PAGESIZE(a));
3854832011-11-18Arne Goedeke 
41889b2012-01-10Arne Goedeke  // we know that p == a->last_free a->last_free = NULL;
68bb052012-02-12Arne Goedeke #endif
3854832011-11-18Arne Goedeke 
dfad0b2012-02-12Arne Goedeke  DOUBLE_UNLINK(a->first, p);
3854832011-11-18Arne Goedeke 
d2892b2011-12-25Arne Goedeke #ifdef BA_DEBUG
41889b2012-01-10Arne Goedeke  memset(p, 0, sizeof(struct ba_page)); if (a->first == p) { fprintf(stderr, "a->first will be old removed page %p\n", a->first);
d2892b2011-12-25Arne Goedeke  } #endif
3854832011-11-18Arne Goedeke  a->num_pages--;
335e6a2012-01-04Arne Goedeke  free(p);
6fc6ce2011-12-30Arne Goedeke  #ifdef BA_DEBUG ba_check_allocator(a, "ba_remove_page", __FILE__, __LINE__); #endif
68bb052012-02-12Arne Goedeke #ifndef BA_USE_MEMALIGN
e2499f2012-01-03Arne Goedeke  if (a->allocated > BA_ALLOC_INITIAL && a->num_pages <= (a->allocated >> 2)) {
689a782012-02-08Arne Goedeke  ba_htable_shrink(a);
6fc6ce2011-12-30Arne Goedeke  }
e2499f2012-01-03Arne Goedeke #endif
a5e1b82011-11-09Arne Goedeke }