pike.git / src / block_allocator.c

version» Context lines:

pike.git/src/block_allocator.c:30:   static INLINE void ba_htable_grow(struct block_allocator * a);   #endif      #define BA_NBLOCK(a, p, ptr) ((uintptr_t)((char*)ptr - (char)(p+1))/(a)->block_size)      #define BA_DIVIDE(a, b) ((a) / (b) + (!!((a) & ((b)-1))))   #define BA_PAGESIZE(a) (sizeof(struct ba_page) + (a)->blocks * (a)->block_size)   #define BA_HASH_MASK(a) (((a->allocated)) - 1)      #ifdef BA_HASH_THLD - # define BA_BYTES(a) ( (sizeof(ba_page) + ((a->allocated > BA_HASH_THLD) ? sizeof(ba_page_t) : 0)) * ((a)->allocated) ) + # define BA_BYTES(a) ( (sizeof(ba_page) + ((a->allocated > BA_HASH_THLD) ? sizeof(uint32_t) : 0)) * ((a)->allocated) )   #else   # define BA_BYTES(a) ( (sizeof(ba_page) * ((a)->allocated) )   #endif      #define PRINT_NODE(a, name) do {\    fprintf(stderr, #name": %p\n", a->name);\   } while (0)         PMOD_EXPORT void ba_show_pages(const struct block_allocator * a) {
pike.git/src/block_allocator.c:54:    fprintf(stderr, "allocated: %u\n", a->allocated);   #endif    fprintf(stderr, "num_pages: %u\n", a->num_pages);    fprintf(stderr, "max_empty_pages: %u\n", a->max_empty_pages);    fprintf(stderr, "empty_pages: %u\n", a->empty_pages);    fprintf(stderr, "magnitude: %u\n", a->magnitude);    PRINT_NODE(a, empty);    PRINT_NODE(a, first);   #ifdef BA_USE_MEMALIGN    PRINT_NODE(a, full); +  PRINT_NODE(a, alloc);   #else    PRINT_NODE(a, last_free);   #endif       PAGE_LOOP(a, { -  ba_block_t blocks_used; +  uint32_t blocks_used;    blocks_used = p->used;    fprintf(stderr, "(%p)\t%f\t(%u %d) --> (prev: %p | next: %p)\n",    p, blocks_used/(double)a->blocks * 100,    blocks_used,    blocks_used,    p->prev, p->next);    });   }      #ifdef BA_STATS
pike.git/src/block_allocator.c:127:    PRCOUNT(find_hash);    PRCOUNT(free_empty);    PRCOUNT(free_full);    fprintf(stat_file, " all %lu\n", (long unsigned int)all);   }   #endif      //#define BA_ALIGNMENT 8      PMOD_EXPORT INLINE void ba_init(struct block_allocator * a, -  uint32_t block_size, ba_page_t blocks) { +  uint32_t block_size, uint32_t blocks) {    uint32_t page_size;      #ifdef BA_ALIGNMENT    if (block_size & (BA_ALIGNMENT - 1))    block_size += (BA_ALIGNMENT - (block_size & (BA_ALIGNMENT - 1)));   #endif       page_size = block_size * blocks + BLOCK_HEADER_SIZE;       a->empty = a->first = NULL;
pike.git/src/block_allocator.c:288: Inside #if defined(BA_STATS)
   a->num_pages = 0;   #ifdef BA_STATS    a->stats.st_max = a->stats.st_used = 0;   #endif   }      #ifndef BA_USE_MEMALIGN   static INLINE void ba_grow(struct block_allocator * a) {    if (a->allocated) {    // try to detect 32bit overrun? -  if (a->allocated >= ((ba_page_t)1 << (sizeof(ba_page_t)*8-1))) { +  if (a->allocated >= ((uint32_t)1 << (sizeof(uint32_t)*8-1))) {    BA_ERROR("too many pages.\n");    }    ba_htable_grow(a);    } else {    ba_init(a, a->block_size, a->blocks);    }   #ifdef BA_DEBUG    ba_check_allocator(a, "ba_grow", __FILE__, __LINE__);   #endif   }      #define MIX(t) do { \    t ^= (t >> 20) ^ (t >> 12); \    t ^= (t >> 7) ^ (t >> 4); \   } while(0)    - static INLINE ba_page_t hash1(const struct block_allocator * a, + static INLINE uint32_t hash1(const struct block_allocator * a,    const void * ptr) {    uintptr_t t = ((uintptr_t)ptr) >> a->magnitude;       MIX(t);    -  return (ba_page_t) t; +  return (uint32_t) t;   }    - static INLINE ba_page_t hash2(const struct block_allocator * a, + static INLINE uint32_t hash2(const struct block_allocator * a,    const void * ptr) {    uintptr_t t = ((uintptr_t)ptr) >> a->magnitude;       t++;    MIX(t);    -  return (ba_page_t) t; +  return (uint32_t) t;   }      static INLINE void ba_htable_grow(struct block_allocator * a) { -  ba_page_t n, old; +  uint32_t n, old;    old = a->allocated;    a->allocated *= 2;    a->pages = (ba_page*)realloc(a->pages, a->allocated * sizeof(ba_page));    if (!a->pages) Pike_error("nomem");    memset(a->pages+old, 0, old * sizeof(ba_page));    for (n = 0; n < old; n++) {    ba_page * b = a->pages + n;       while (*b) { -  ba_page_t h = hash1(a, BA_LASTBLOCK(a, *b)) & BA_HASH_MASK(a); +  uint32_t h = hash1(a, BA_LASTBLOCK(a, *b)) & BA_HASH_MASK(a);    if (h != n) {    ba_page p = *b;    *b = p->hchain;    p->hchain = a->pages[h];    a->pages[h] = p;    continue;    }    b = &(*b)->hchain;    }    }   }      static INLINE void ba_htable_shrink(struct block_allocator * a) { -  ba_page_t n; +  uint32_t n;       a->allocated /= 2;       for (n = 0; n < a->allocated; n++) {    ba_page p = a->pages[a->allocated + n];       if (p) {    ba_page t = p;    if (a->pages[n]) {    while (t->hchain) t = t->hchain;
pike.git/src/block_allocator.c:376:    a->pages = (ba_page*)realloc(a->pages, a->allocated * sizeof(ba_page));    if (!a->pages) Pike_error("nomem");   }      #ifdef BA_DEBUG   PMOD_EXPORT void ba_print_htable(const struct block_allocator * a) {    unsigned int i;       for (i = 0; i < a->allocated; i++) {    ba_page p = a->pages[i]; -  ba_page_t hval; +  uint32_t hval;    if (!p) {    fprintf(stderr, "empty %u\n", i);    }    while (p) {    void * ptr = BA_LASTBLOCK(a, p);    hval = hash1(a, ptr);    fprintf(stderr, "%u\t%X[%u]\t%p-%p\t%X\n", i, hval, hval & BA_HASH_MASK(a), p+1, ptr, (unsigned int)((uintptr_t)ptr >> a->magnitude));    p = p->hchain;    }    }   }   #endif      /*    * insert the pointer to an allocated page into the    * hashtable. uses linear probing and open allocation.    */   static INLINE void ba_htable_insert(const struct block_allocator * a,    ba_page p) { -  ba_page_t hval = hash1(a, BA_LASTBLOCK(a, p)); +  uint32_t hval = hash1(a, BA_LASTBLOCK(a, p));    ba_page * b = a->pages + (hval & BA_HASH_MASK(a));         #ifdef BA_DEBUG    while (*b) {    if (*b == p) {    fprintf(stderr, "inserting (%p) twice\n", p);    fprintf(stderr, "is (%p)\n", *b);    BA_ERROR("double insert.\n");    return;
pike.git/src/block_allocator.c:425:    hval & BA_HASH_MASK(a), *b, n);   #endif       p->hchain = *b;    *b = p;   }         #if 0   static INLINE void ba_htable_replace(const struct block_allocator * a, -  const void * ptr, const ba_page_t n, -  const ba_page_t new) { -  ba_page_t hval = hash1(a, ptr); -  ba_page_t * b = a->htable + (hval & BA_HASH_MASK(a)); +  const void * ptr, const uint32_t n, +  const uint32_t new) { +  uint32_t hval = hash1(a, ptr); +  uint32_t * b = a->htable + (hval & BA_HASH_MASK(a));       while (*b) {    if (*b == n) {    *b = new;    BA_PAGE(a, new)->hchain = BA_PAGE(a, n)->hchain;    BA_PAGE(a, n)->hchain = 0;    return;    }    b = &BA_PAGE(a, *b)->hchain;    }
pike.git/src/block_allocator.c:451: Inside #if 0 and #if defined(DEBUG)
   fprintf(stderr, "ba_htable_replace(%p, %u, %u)\n", ptr, n, new);    fprintf(stderr, "hval: %u, %u, %u\n", hval, hval & BA_HASH_MASK(a), BA_HASH_MASK(a));    ba_print_htable(a);    BA_ERROR("did not find index to replace.\n")   #endif   }   #endif      static INLINE void ba_htable_delete(const struct block_allocator * a,    ba_page p) { -  ba_page_t hval = hash1(a, BA_LASTBLOCK(a, p)); +  uint32_t hval = hash1(a, BA_LASTBLOCK(a, p));    ba_page * b = a->pages + (hval & BA_HASH_MASK(a));       while (*b) {    if (*b == p) {    *b = p->hchain;    p->hchain = 0;    return;    }    b = &(*b)->hchain;    }   #ifdef DEBUG    ba_print_htable(a);    fprintf(stderr, "ba_htable_delete(%p, %u)\n", p, n);    BA_ERROR("did not find index to delete.\n")   #endif   }      static INLINE void ba_htable_lookup(struct block_allocator * a,    const void * ptr) {    ba_page p; -  ba_page_t h[2]; +  uint32_t h[2];    unsigned char c, b = 0;    h[0] = hash1(a, ptr);    h[1] = hash2(a, ptr);       c = ((uintptr_t)ptr >> (a->magnitude - 1)) & 1;      LOOKUP:    p = a->pages[h[c] & BA_HASH_MASK(a)];    while (p) {    if (BA_CHECK_PTR(a, p, ptr)) {
pike.git/src/block_allocator.c:496:    p = p->hchain;    }    c = !c;    if (!(b++)) goto LOOKUP;   }   #endif      #ifdef BA_DEBUG   PMOD_EXPORT INLINE void ba_check_allocator(struct block_allocator * a,    char *fun, char *file, int line) { -  ba_page_t n; +  uint32_t n;    int bad = 0;    ba_page p;       if (a->empty_pages > a->num_pages) {    fprintf(stderr, "too many empty pages.\n");    bad = 1;    } -  + #ifdef BA_USE_MEMALIGN +  if (a->alloc) { +  if (a->alloc == a->first) { +  fprintf(stderr, "alloc is first\n"); +  bad = 1; +  } +  if (a->alloc == a->full) { +  fprintf(stderr, "alloc is full\n"); +  bad = 1; +  } +  if (a->alloc == a->empty) { +  fprintf(stderr, "alloc is empty\n"); +  bad = 1; +  } +  }    -  +  if (a->empty && a->empty == a->full) { +  fprintf(stderr, "full is empty\n"); +  bad = 1; +  } +  +  p = a->empty; +  while (p) { +  if (p->used) { +  fprintf(stderr, "free is not empty\n"); +  bad = 1; +  } +  p = p->next; +  } +  p = a->full; +  while (p) { +  if (p->used != a->blocks) { +  fprintf(stderr, "full is not full\n"); +  bad = 1; +  } +  if (p->first) { +  fprintf(stderr, "full has free block: %p\n", p->first); +  bad = 1; +  } +  p = p->next; +  } +  +  n = 0; +  PAGE_LOOP(a, { +  if (n++ >= a->num_pages) { +  fprintf(stderr, "broken lists (found %u of %u pages)\n", +  n, a->num_pages); +  bad = 1; +  break; +  } +  }); + #endif +    #ifndef BA_USE_MEMALIGN    for (n = 0; n < a->allocated; n++) { -  ba_page_t hval; +  uint32_t hval;    int found = 0;    p = a->pages[n];       while (p) {    if (!p->first && p != a->first) {    if (p->prev || p->next) {    fprintf(stderr, "page %p is full but in list. next: %p"    "prev: %p\n",    p, p->next,    p->prev);
pike.git/src/block_allocator.c:567:       a->num_pages++;   #ifndef BA_USE_MEMALIGN    p = (ba_page)malloc(BA_PAGESIZE(a));   #else    p = (ba_page)memalign((size_t)(1 << a->magnitude), BA_PAGESIZE(a));   #endif    if (!p) BA_ERROR("no mem. alloc returned zero.");    ba_free_page(a, p);    p->next = p->prev = NULL; -  a->first = p; +  a->alloc = p;   #ifndef BA_USE_MEMALIGN    ba_htable_insert(a, p);   #endif   #ifdef BA_DEBUG    ba_check_allocator(a, "ba_alloc after insert", __FILE__, __LINE__);   #endif   #ifdef BA_DEBUG    PIKE_MEM_RW(BA_LASTBLOCK(a, p)->magic);    BA_LASTBLOCK(a, p)->magic = BA_MARK_FREE;    PIKE_MEM_RW(BA_BLOCKN(a, p, 0)->magic);    BA_BLOCKN(a, p, 0)->magic = BA_MARK_ALLOC; -  ba_check_allocator(a, "ba_alloc after insert", __FILE__, __LINE__); +  ba_check_allocator(a, "ba_alloc after insert.", __FILE__, __LINE__);   #endif   }      PMOD_EXPORT void ba_low_alloc(struct block_allocator * a) {    //fprintf(stderr, "ba_alloc(%p)\n", a);   #ifdef BA_DEBUG    ba_check_allocator(a, "ba_alloc top", __FILE__, __LINE__);   #endif -  -  //fprintf(stderr, "allocating new page. was %p\n", p); -  if (a->first) { -  ba_page p = a->first; -  DOUBLE_SHIFT(a->first); +  if (a->alloc) { +  a->alloc->first = NULL; +  a->alloc->used = a->blocks;   #ifdef BA_USE_MEMALIGN -  DOUBLE_LINK(a->full, p); - #else -  p->next = NULL; +  DOUBLE_LINK(a->full, a->alloc);   #endif -  p->first = NULL; +     }    -  if (a->first) return; -  -  if (a->empty_pages) { -  a->first = a->empty; +  if (a->first) { +  a->alloc = a->first; +  DOUBLE_SHIFT(a->first); +  } else if (a->empty) { +  a->alloc = a->empty;    SINGLE_SHIFT(a->empty);    a->empty_pages--; -  a->first->next = NULL; -  } else ba_alloc_page(a); +  } else { +  a->alloc = NULL; +  ba_alloc_page(a); +  }    -  +  a->free_blk = a->alloc->first; +    #ifdef BA_DEBUG -  if (!a->first->first) { +  if (!a->free_blk) {    ba_show_pages(a);    BA_ERROR("a->first has no first block!\n");    }   #endif      #ifdef BA_DEBUG    ba_check_allocator(a, "ba_alloc after grow", __FILE__, __LINE__);   #endif   }      PMOD_EXPORT void ba_low_free(struct block_allocator * a, ba_page p,    ba_block_header ptr) { -  // page was full -  if (unlikely(!ptr->next)) { -  INC(free_full); - #ifdef BA_USE_MEMALIGN -  DOUBLE_UNLINK(a->full, p); - #endif - #if 1 -  if (a->first) { -  p->prev = a->first; -  p->next = a->first->next; -  a->first->next = p; -  if (p->next) p->next->prev = p; -  } else - #endif -  DOUBLE_LINK(a->first, p); -  } else if (!p->used) { +  if (!p->used) {    INC(free_empty);    if (a->empty_pages == a->max_empty_pages) {    ba_remove_page(a, p);    return;    }    DOUBLE_UNLINK(a->first, p);    SINGLE_LINK(a->empty, p);    a->empty_pages ++; -  +  } else { // page was full +  INC(free_full); + #ifdef BA_USE_MEMALIGN +  DOUBLE_UNLINK(a->full, p); + #endif +  DOUBLE_LINK(a->first, p);    }   }      #ifndef BA_USE_MEMALIGN   static INLINE void ba_htable_linear_lookup(struct block_allocator * a,    const void * ptr) {    PAGE_LOOP(a, {    if (BA_CHECK_PTR(a, p, ptr)) {    a->last_free = p;    return;