0a3d34 | 2011-11-10 | Arne Goedeke | | #include <stdlib.h>
|
67795d | 2011-11-11 | Arne Goedeke | | #include <string.h>
|
385483 | 2011-11-18 | Arne Goedeke | | #include <stdint.h>
|
0a3d34 | 2011-11-10 | Arne Goedeke | |
|
a60c24 | 2011-12-25 | Arne Goedeke | | #include "bitvector.h"
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef PMOD_EXPORT
#include "pike_error.h"
#else
#define PMOD_EXPORT
#include <stdio.h>
#include <unistd.h>
void _Pike_error(int line, char * file, char * msg) {
fprintf(stderr, "%s:%d\t%s\n", file, line, msg);
_exit(1);
}
#define Pike_error(x) _Pike_error(__LINE__, __FILE__, x);
|
e90c49 | 2011-11-27 | Arne Goedeke | | #endif
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef HAS___BUILTIN_EXPECT
#define likely(x) (__builtin_expect((x), 1))
#define unlikely(x) (__builtin_expect((x), 0))
#else
#define likely(x) (x)
#define unlikely(x) (x)
#endif
#include "block_allocator.h"
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
struct ba_page {
char * data;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ba_page_t next, prev;
ba_page_t hchain;
|
385483 | 2011-11-18 | Arne Goedeke | | #ifdef BA_SEGREGATE
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ba_block_t blocks_used, first;
|
385483 | 2011-11-18 | Arne Goedeke | | #else
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ba_block_t blocks_used;
|
67795d | 2011-11-11 | Arne Goedeke | | uint16_t free_mask;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | uintptr_t mask[1];
|
385483 | 2011-11-18 | Arne Goedeke | | #endif
};
struct ba_block_header {
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef BA_DEBUG
uint32_t magic;
#endif
#ifdef BA_SEGREGATE
#include "block_allocator.h"
ba_block_t next;
#endif
|
0a3d34 | 2011-11-10 | Arne Goedeke | | };
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
#define BA_MARK_FREE 0xF4337575
#define BA_MARK_ALLOC 0x4110C375
#define BA_ALLOC_INITIAL 4
#define BA_HASH_THLD 4
#ifdef BA_HASH_THLD
#define IF_HASH(x) do { if(a->allocated <= BA_HASH_THLD) break; x; }\
while(0)
#else
#define IF_HASH(x) x
|
385483 | 2011-11-18 | Arne Goedeke | | #endif
|
0a3d34 | 2011-11-10 | Arne Goedeke | |
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
fdfd28 | 2011-12-25 | Arne Goedeke | | size_t _malloc_counter = 0;
|
67795d | 2011-11-11 | Arne Goedeke | | static inline void ba_htable_insert(const struct block_allocator * a,
|
6fc6ce | 2011-12-30 | Arne Goedeke | | const void * ptr, const ba_page_t n);
|
385483 | 2011-11-18 | Arne Goedeke | | static inline void ba_remove_page(struct block_allocator * a,
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ba_page_t n);
typedef struct ba_block_header * ba_block_header;
|
67795d | 2011-11-11 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #define BA_BLOCKN(a, p, n) ((ba_block_header)(((p)->data) + (n)*((a)->block_size)))
|
e90c49 | 2011-11-27 | Arne Goedeke | | #define BA_LASTBLOCK(a, p) BA_BLOCKN(a, p, (a)->blocks - 1)
#define BA_NBLOCK(a, p, ptr) ((uintptr_t)((char*)ptr - (p)->data)/(a)->block_size)
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #define BA_DIVIDE(a, b) ((a) / (b) + (!!((a) & ((b)-1))))
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | #define BA_PAGESIZE(a) ((a)->blocks * (a)->block_size)
|
385483 | 2011-11-18 | Arne Goedeke | | #define BA_SPAGE_SIZE(a) (sizeof(struct ba_page) + (BA_DIVIDE((a)->blocks, sizeof(uintptr_t)*8) - 1)*sizeof(uintptr_t))
#define BA_HASH_MASK(a) (((a->allocated)) - 1)
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #define BA_CHECK_PTR(a, p, ptr) ((char*)ptr >= p->data && (void*)BA_LASTBLOCK(a,p) >= ptr)
|
385483 | 2011-11-18 | Arne Goedeke | |
#ifdef BA_SEGREGATE
# define BA_PAGE(a, n) ((a)->pages + (n) - 1)
|
6fc6ce | 2011-12-30 | Arne Goedeke | | # define BA_BYTES(a) ( (sizeof(struct ba_page) + sizeof(ba_page_t)) * ((a)->allocated) )
|
385483 | 2011-11-18 | Arne Goedeke | | #else
# define BA_PAGE(a, n) ((ba_page)((char*)(a)->pages + ((n)-1) * ((a)->ba_page_size)))
|
6fc6ce | 2011-12-30 | Arne Goedeke | | # define BA_BYTES(a) (( (a)->ba_page_size + sizeof(ba_page_t)) * ((a)->allocated))
|
385483 | 2011-11-18 | Arne Goedeke | | # define BA_LAST_BITS(a) ((a)->blocks & (sizeof(uintptr_t)*8 - 1))
|
6fc6ce | 2011-12-30 | Arne Goedeke | | # define BA_LAST_MASK(a) (BA_LAST_BITS(a) ? MASK(uintptr_t, BA_LAST_BITS(a)) : ~((uintptr_t)0))
|
385483 | 2011-11-18 | Arne Goedeke | | # define BA_MASK_NUM(a) (((a)->ba_page_size - sizeof(struct ba_page) + sizeof(uintptr_t))/sizeof(uintptr_t))
#endif
|
67795d | 2011-11-11 | Arne Goedeke | |
static inline void ba_realloc(struct block_allocator * a) {
|
475075 | 2011-11-19 | Arne Goedeke | | unsigned int i;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | void *old;
ba_page p;
a->pages = realloc(old = a->pages, BA_BYTES(a));
|
67795d | 2011-11-11 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | if (unlikely(!a->pages)) {
a->pages = old;
fprintf(stderr, "realloc(%lu) failed.\n", BA_BYTES(a));
|
67795d | 2011-11-11 | Arne Goedeke | | Pike_error("no mem");
}
|
385483 | 2011-11-18 | Arne Goedeke | | #ifndef BA_SEGREGATE
memset((void*)BA_PAGE(a, a->num_pages+1), 0, BA_BYTES(a) - a->ba_page_size*a->num_pages);
#else
memset((void*)BA_PAGE(a, a->num_pages+1), 0, BA_BYTES(a) - sizeof(struct ba_page)*a->num_pages);
#endif
|
6fc6ce | 2011-12-30 | Arne Goedeke | | IF_HASH(
a->htable = (ba_page_t*) BA_PAGE(a, (a->allocated)+1);
for (i = 0; i < a->num_pages; i++) {
p = BA_PAGE(a, i+1);
p->hchain = 0;
ba_htable_insert(a, BA_LASTBLOCK(a, p), i+1);
}
);
#ifdef BA_DEBUG
ba_check_allocator(a, "realloc", __FILE__, __LINE__);
#endif
|
67795d | 2011-11-11 | Arne Goedeke | | }
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | PMOD_EXPORT void ba_init(struct block_allocator * a,
uint32_t block_size, ba_page_t blocks) {
|
475075 | 2011-11-19 | Arne Goedeke | | uint32_t page_size = block_size * blocks;
|
385483 | 2011-11-18 | Arne Goedeke | |
if (blocks > 0xfffe) {
Pike_error("number of blocks cannot exceed 2^16-1\n");
}
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | a->first = 0;
|
67795d | 2011-11-11 | Arne Goedeke | | a->last_free = 0;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
0a3d34 | 2011-11-10 | Arne Goedeke | | if ((page_size & (page_size - 1)) == 0)
|
67795d | 2011-11-11 | Arne Goedeke | | a->magnitude = (uint16_t)ctz32(page_size);
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | else
|
67795d | 2011-11-11 | Arne Goedeke | | a->magnitude = (uint16_t)ctz32(round_up32(page_size));
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
0a3d34 | 2011-11-10 | Arne Goedeke | | a->block_size = block_size;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | a->blocks = blocks;
a->num_pages = 0;
|
938726 | 2011-12-25 | Arne Goedeke | | a->empty_pages = 0;
a->max_empty_pages = 3;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
67795d | 2011-11-11 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | a->allocated = BA_ALLOC_INITIAL;
|
67795d | 2011-11-11 | Arne Goedeke | | a->pages = NULL;
|
385483 | 2011-11-18 | Arne Goedeke | | #ifndef BA_SEGREGATE
a->ba_page_size = BA_SPAGE_SIZE(a);
#endif
|
67795d | 2011-11-11 | Arne Goedeke | | ba_realloc(a);
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | | PMOD_EXPORT void ba_free_all(struct block_allocator * a) {
|
475075 | 2011-11-19 | Arne Goedeke | | unsigned int i;
|
67795d | 2011-11-11 | Arne Goedeke | |
for (i = 0; i < a->num_pages; i++) {
|
475075 | 2011-11-19 | Arne Goedeke | | free(BA_PAGE(a, i+1)->data);
|
6fc6ce | 2011-12-30 | Arne Goedeke | | BA_PAGE(a, i+1)->data = NULL;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | | IF_HASH(
memset(a->htable, 0, a->allocated * sizeof(ba_page_t));
);
|
67795d | 2011-11-11 | Arne Goedeke | | a->num_pages = 0;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | a->empty_pages = 0;
a->first = 0;
|
67795d | 2011-11-11 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | | PMOD_EXPORT void ba_count_all(struct block_allocator * a, size_t *num, size_t *size) {
|
475075 | 2011-11-19 | Arne Goedeke | | unsigned int i;
|
385483 | 2011-11-18 | Arne Goedeke | | size_t n = 0;
|
e90c49 | 2011-11-27 | Arne Goedeke | | fprintf(stderr, "page_size: %u, pages: %u\n", BA_PAGESIZE(a), a->num_pages);
|
385483 | 2011-11-18 | Arne Goedeke | | *size = BA_BYTES(a) + a->num_pages * BA_PAGESIZE(a);
for (i = 0; i < a->num_pages; i++) {
|
475075 | 2011-11-19 | Arne Goedeke | | n += BA_PAGE(a, i+1)->blocks_used;
|
385483 | 2011-11-18 | Arne Goedeke | | }
*num = n;
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | | PMOD_EXPORT void ba_destroy(struct block_allocator * a) {
|
67795d | 2011-11-11 | Arne Goedeke | | ba_free_all(a);
free(a->pages);
|
6fc6ce | 2011-12-30 | Arne Goedeke | | a->allocated = 0;
|
67795d | 2011-11-11 | Arne Goedeke | | a->pages = NULL;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
67795d | 2011-11-11 | Arne Goedeke | | static inline void ba_grow(struct block_allocator * a) {
|
385483 | 2011-11-18 | Arne Goedeke | | if (a->allocated) {
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
if (a->allocated >= ((ba_page_t)1 << (sizeof(ba_page_t)*8-1))) {
Pike_error("too many pages.\n");
}
|
385483 | 2011-11-18 | Arne Goedeke | | a->allocated *= 2;
ba_realloc(a);
} else {
ba_init(a, a->block_size, a->blocks);
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef BA_DEBUG
ba_check_allocator(a, "ba_grow", __FILE__, __LINE__);
#endif
|
67795d | 2011-11-11 | Arne Goedeke | | }
static inline void ba_shrink(struct block_allocator * a) {
a->allocated /= 2;
ba_realloc(a);
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #define MIX(t) do { \
t ^= (t >> 20) ^ (t >> 12); \
t ^= (t >> 7) ^ (t >> 4); \
} while(0)
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | static inline ba_page_t hash1(const struct block_allocator * a,
|
67795d | 2011-11-11 | Arne Goedeke | | const void * ptr) {
uintptr_t t = ((uintptr_t)ptr) >> a->magnitude;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | MIX(t);
return (ba_page_t) t;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | | static inline ba_page_t hash2(const struct block_allocator * a,
|
67795d | 2011-11-11 | Arne Goedeke | | const void * ptr) {
uintptr_t t = ((uintptr_t)ptr) >> a->magnitude;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | t++;
MIX(t);
return (ba_page_t) t;
}
void ba_print_htable(const struct block_allocator * a) {
unsigned int i;
ba_page p;
fprintf(stderr, "allocated: %u\n", a->allocated);
fprintf(stderr, "num_pages: %u\n", a->num_pages);
fprintf(stderr, "max_empty_pages: %u\n", a->max_empty_pages);
fprintf(stderr, "empty_pages: %u\n", a->empty_pages);
fprintf(stderr, "magnitude: %u\n", a->magnitude);
fprintf(stderr, "block_size: %u\n", a->block_size);
for (i = 0; i < a->allocated; i++) {
ba_page_t n = a->htable[i];
ba_page_t hval;
void * ptr;
while (n) {
p = BA_PAGE(a, n);
ptr = BA_LASTBLOCK(a, p);
hval = hash1(a, ptr);
fprintf(stderr, "%u\t%X\t%p-%p\t%X (page %d)\n", i, hval, p->data, ptr, (unsigned int)((uintptr_t)ptr >> a->magnitude), n);
n = p->hchain;
}
}
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
67795d | 2011-11-11 | Arne Goedeke | | static inline void ba_htable_insert(const struct block_allocator * a,
|
6fc6ce | 2011-12-30 | Arne Goedeke | | const void * ptr, const ba_page_t n) {
ba_page_t hval = hash1(a, ptr);
ba_page_t * b = a->htable + (hval & BA_HASH_MASK(a));
#ifdef BA_DEBUG
while (*b) {
if (*b == n) {
fprintf(stderr, "inserting (%p, %d) twice\n", ptr, n);
fprintf(stderr, "is (%p, %d)\n", BA_PAGE(a, n)->data, n);
Pike_error("double insert.\n");
return;
}
b = &BA_PAGE(a, *b)->hchain;
}
b = a->htable + (hval & BA_HASH_MASK(a));
#endif
|
385483 | 2011-11-18 | Arne Goedeke | |
BA_PAGE(a, n)->hchain = *b;
*b = n;
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | | static inline void ba_htable_replace(const struct block_allocator * a,
|
6fc6ce | 2011-12-30 | Arne Goedeke | | const void * ptr, const ba_page_t n,
const ba_page_t new) {
ba_page_t hval = hash1(a, ptr);
ba_page_t * b = a->htable + (hval & BA_HASH_MASK(a));
|
385483 | 2011-11-18 | Arne Goedeke | |
while (*b) {
if (*b == n) {
*b = new;
BA_PAGE(a, new)->hchain = BA_PAGE(a, n)->hchain;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | BA_PAGE(a, n)->hchain = 0;
|
385483 | 2011-11-18 | Arne Goedeke | | return;
}
b = &BA_PAGE(a, *b)->hchain;
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef DEBUG
fprintf(stderr, "ba_htable_replace(%p, %u, %u)\n", ptr, n, new);
fprintf(stderr, "hval: %u, %u, %u\n", hval, hval & BA_HASH_MASK(a), BA_HASH_MASK(a));
ba_print_htable(a);
Pike_error("did not find index to replace.\n")
#endif
|
385483 | 2011-11-18 | Arne Goedeke | | }
static inline void ba_htable_delete(const struct block_allocator * a,
|
6fc6ce | 2011-12-30 | Arne Goedeke | | const void * ptr, const ba_page_t n) {
ba_page_t hval = hash1(a, ptr);
ba_page_t * b = a->htable + (hval & BA_HASH_MASK(a));
|
385483 | 2011-11-18 | Arne Goedeke | |
while (*b) {
if (*b == n) {
*b = BA_PAGE(a, n)->hchain;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | BA_PAGE(a, n)->hchain = 0;
|
385483 | 2011-11-18 | Arne Goedeke | | return;
}
b = &BA_PAGE(a, *b)->hchain;
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef DEBUG
ba_print_htable(a);
fprintf(stderr, "ba_htable_delete(%p, %u)\n", ptr, n);
Pike_error("did not find index to delete.\n")
#endif
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | | static inline ba_page_t ba_htable_lookup(const struct block_allocator * a,
const void * ptr) {
int c = 0;
ba_page_t n1, n2;
|
0a3d34 | 2011-11-10 | Arne Goedeke | | ba_page p;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | n1 = a->htable[hash1(a, ptr) & BA_HASH_MASK(a)];
while (n1) {
p = BA_PAGE(a, n1);
if (BA_CHECK_PTR(a, p, ptr)) {
return n1;
}
if (c++ > 100) {
n2 = a->htable[hash1(a, ptr) & BA_HASH_MASK(a)];
n1 = n2;
do {
p = BA_PAGE(a, n1);
fprintf(stderr, "%d %p -> %d\n", n1, p->data, p->hchain);
if (n1 == p->hchain) break;
n1 = p->hchain;
} while (n2 != n1);
Pike_error("hash chain is infinite\n");
|
67795d | 2011-11-11 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | | n1 = p->hchain;
}
n2 = a->htable[hash2(a, ptr) & BA_HASH_MASK(a)];
while (n2) {
p = BA_PAGE(a, n2);
if (BA_CHECK_PTR(a, p, ptr)) {
return n2;
}
if (c++ > 100) {
Pike_error("hash chain is infinite\n");
}
n2 = p->hchain;
|
67795d | 2011-11-11 | Arne Goedeke | | }
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | return 0;
}
|
67795d | 2011-11-11 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | PMOD_EXPORT void ba_check_allocator(struct block_allocator * a,
char *fun, char *file, int line) {
unsigned int i = 0;
int bad = 0;
|
67795d | 2011-11-11 | Arne Goedeke | | ba_page p;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | if (a->empty_pages > a->num_pages) {
fprintf(stderr, "too many empty pages.\n");
bad = 1;
}
for (i = 1; i <= a->num_pages; i++) {
ba_page_t hval;
int found = 0;
ba_page_t n;
p = BA_PAGE(a, i);
if (p->blocks_used == a->blocks) {
if (p->prev || p->next) {
fprintf(stderr, "block is full but in list. next: %u prev: %u",
p->next, p->prev);
bad = 1;
|
67795d | 2011-11-11 | Arne Goedeke | | }
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
IF_HASH(
hval = hash1(a, BA_LASTBLOCK(a, p));
n = a->htable[hval & BA_HASH_MASK(a)];
while (n) {
if (n == i) {
found = 1;
break;
}
n = BA_PAGE(a, n)->hchain;
|
67795d | 2011-11-11 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
if (!found) {
fprintf(stderr, "did not find page %d, %p (%u) in htable.\n",
i, BA_LASTBLOCK(a, BA_PAGE(a, i)), hval);
fprintf(stderr, "looking in bucket %u mask: %d\n", hval & BA_HASH_MASK(a), BA_HASH_MASK(a));
bad = 1;
}
for (i = 0; i < a->allocated; i++) {
n = a->htable[i];
while (n) {
p = BA_PAGE(a, n);
hval = hash1(a, BA_LASTBLOCK(a, p));
if (i != (hval & BA_HASH_MASK(a))) {
fprintf(stderr, "page %u found in wrong bucket %d\n",
n, i);
bad = 1;
}
n = p->hchain;
}
}
);
}
if (bad) {
ba_print_htable(a);
fprintf(stderr, "\nCalled from %s:%d:%s\n", fun, line, file);
fprintf(stderr, "pages: %u\n", a->num_pages);
Pike_error("bad");
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
PMOD_EXPORT void * ba_alloc(struct block_allocator * a) {
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | ba_page p;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef BA_DEBUG
ba_check_allocator(a, "ba_alloc top", __FILE__, __LINE__);
#endif
|
385483 | 2011-11-18 | Arne Goedeke | | #ifndef BA_SEGREGATE
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | size_t i, j;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #else
int i;
|
385483 | 2011-11-18 | Arne Goedeke | | #endif
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | | if (likely(a->first)) {
#ifndef BA_SEGREGATE
|
67795d | 2011-11-11 | Arne Goedeke | | uintptr_t m;
|
385483 | 2011-11-18 | Arne Goedeke | | #endif
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ba_block_header ptr;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
d2892b | 2011-12-25 | Arne Goedeke | | #ifdef BA_DEBUG
if (a->first > a->num_pages) {
fprintf(stderr, "unused page set. %d > num_pages: %d.\n", a->first, a->num_pages);
}
#endif
|
385483 | 2011-11-18 | Arne Goedeke | | p = BA_PAGE(a, a->first);
|
67795d | 2011-11-11 | Arne Goedeke | |
|
d2892b | 2011-12-25 | Arne Goedeke | | #ifdef BA_DEBUG
if (p->prev) {
fprintf(stderr, "a->first has previous: %d\n", p->prev);
}
#endif
|
385483 | 2011-11-18 | Arne Goedeke | | #ifndef BA_SEGREGATE
|
67795d | 2011-11-11 | Arne Goedeke | | i = p->free_mask;
|
d2892b | 2011-12-25 | Arne Goedeke | | #ifdef BA_DEBUG
|
385483 | 2011-11-18 | Arne Goedeke | | if (i >= BA_MASK_NUM(a)) {
Pike_error("free mask is out of range!\n");
}
|
d2892b | 2011-12-25 | Arne Goedeke | | #endif
|
67795d | 2011-11-11 | Arne Goedeke | | m = p->mask[i];
|
d2892b | 2011-12-25 | Arne Goedeke | | #ifdef BA_DEBUG
|
67795d | 2011-11-11 | Arne Goedeke | | if (!m) {
|
f25b0b | 2011-11-10 | Arne Goedeke | | Pike_error("This should not happen!\n");
}
|
d2892b | 2011-12-25 | Arne Goedeke | | #endif
|
67795d | 2011-11-11 | Arne Goedeke | |
#if SIZEOF_CHAR_P == 8
j = ctz64(m);
#else
j = ctz32(m);
#endif
m ^= TBITMASK(uintptr_t, j);
|
385483 | 2011-11-18 | Arne Goedeke | |
|
67795d | 2011-11-11 | Arne Goedeke | | p->mask[i] = m;
|
385483 | 2011-11-18 | Arne Goedeke | |
p->blocks_used ++;
|
e90c49 | 2011-11-27 | Arne Goedeke | | if (m) {
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ptr = BA_BLOCKN(a, p, (i*sizeof(uintptr_t)*8 + j));
goto RETURN_PTR;
|
e90c49 | 2011-11-27 | Arne Goedeke | | } else {
|
67795d | 2011-11-11 | Arne Goedeke | | for (m = i+1; m < BA_MASK_NUM(a); m++) {
if (p->mask[m]) {
p->free_mask = m;
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ptr = BA_BLOCKN(a, p, (i*sizeof(uintptr_t)*8 + j));
goto RETURN_PTR;
|
67795d | 2011-11-11 | Arne Goedeke | | }
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | | RETURN_PTR:
|
d2892b | 2011-12-25 | Arne Goedeke | | #ifdef BA_DEBUG
|
385483 | 2011-11-18 | Arne Goedeke | | if (p->blocks_used != a->blocks) {
fprintf(stderr, "wrong block count detected: %u vs %u\n", p->blocks_used, a->blocks);
Pike_error("croak\n");
}
|
d2892b | 2011-12-25 | Arne Goedeke | | #endif
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
67795d | 2011-11-11 | Arne Goedeke | | if (a->last == a->first) {
a->first = a->last = 0;
} else {
a->first = p->next;
}
p->next = 0;
p->free_mask = BA_MASK_NUM(a);
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ptr = BA_BLOCKN(a, p, (i*sizeof(uintptr_t)*8 + j));
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
385483 | 2011-11-18 | Arne Goedeke | | #else
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef BA_DEBUG
if (p->blocks_used == a->blocks)
Pike_error("baaad!\n");
#endif
|
385483 | 2011-11-18 | Arne Goedeke | | p->blocks_used ++;
|
d2892b | 2011-12-25 | Arne Goedeke | | #ifdef BA_DEBUG
if (p->first < 1 || p->first > a->blocks) {
fprintf(stderr, "bad index: %d (should be [1..%d]\n", p->first-1, a->blocks);
}
#endif
|
385483 | 2011-11-18 | Arne Goedeke | | ptr = BA_BLOCKN(a, p, p->first-1);
|
d2892b | 2011-12-25 | Arne Goedeke | |
#ifdef BA_DEBUG
if (!p->data) {
fprintf(stderr, "got null pointer from uninitialized page %d of block %d (num_pages %d).\n", a->first, p->first-1, a->num_pages);
}
#endif
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | |
if (unlikely(p->blocks_used == a->blocks)) {
a->first = p->next;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | if (a->first)
|
5df15e | 2011-12-25 | Arne Goedeke | | BA_PAGE(a, a->first)->prev = 0;
|
385483 | 2011-11-18 | Arne Goedeke | | p->next = 0;
} else {
|
6fc6ce | 2011-12-30 | Arne Goedeke | | if (p->blocks_used == 1) a->empty_pages --;
p->first = ptr->next;
}
#endif
#ifdef BA_DEBUG
if (ptr->magic != BA_MARK_FREE) {
fprintf(stderr, "found block with bad magic.\n");
|
385483 | 2011-11-18 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ptr->magic = BA_MARK_ALLOC;
#endif
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | return (void*)ptr;
|
385483 | 2011-11-18 | Arne Goedeke | | }
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | if (unlikely(a->num_pages == a->allocated)) {
|
385483 | 2011-11-18 | Arne Goedeke | | ba_grow(a);
|
6fc6ce | 2011-12-30 | Arne Goedeke | | }
#ifdef BA_DEBUG
ba_check_allocator(a, "ba_alloc after grow", __FILE__, __LINE__);
#endif
|
385483 | 2011-11-18 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | a->num_pages++;
p = BA_PAGE(a, a->num_pages);
#ifdef BA_DEBUG
|
d2892b | 2011-12-25 | Arne Goedeke | | if (p->data) {
|
6fc6ce | 2011-12-30 | Arne Goedeke | | void * new = malloc(BA_PAGESIZE(a));
fprintf(stderr, "reusing unfreed page %d, data: %p -> %p\n", a->num_pages,
p->data, new);
p->data = new;
} else
#endif
|
385483 | 2011-11-18 | Arne Goedeke | | p->data = malloc(BA_PAGESIZE(a));
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef BA_DEBUG
|
fdfd28 | 2011-12-25 | Arne Goedeke | | _malloc_counter++;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #endif
|
385483 | 2011-11-18 | Arne Goedeke | | if (!p->data) {
|
6fc6ce | 2011-12-30 | Arne Goedeke | | Pike_error("no mem. alloc returned zero.");
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
385483 | 2011-11-18 | Arne Goedeke | | p->next = p->prev = 0;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | a->first = a->num_pages;
IF_HASH(
ba_htable_insert(a, BA_LASTBLOCK(a, p), a->num_pages);
#ifdef BA_DEBUG
ba_check_allocator(a, "ba_alloc after insert", __FILE__, __LINE__);
#endif
);
|
385483 | 2011-11-18 | Arne Goedeke | | #ifndef BA_SEGREGATE
|
e90c49 | 2011-11-27 | Arne Goedeke | | if (BA_MASK_NUM(a) > 1)
memset((void*)p->mask, 0xff, (BA_MASK_NUM(a)-1)*sizeof(uintptr_t));
p->mask[BA_MASK_NUM(a)-1] = BA_LAST_MASK(a);
p->mask[0] ^= TBITMASK(uintptr_t, 0);
|
385483 | 2011-11-18 | Arne Goedeke | | p->free_mask = 0;
p->blocks_used = 1;
#else
p->blocks_used = 1;
p->first = 2;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | for (i = 1; i < a->blocks; i++) {
#ifdef BA_DEBUG
BA_BLOCKN(a, p, i)->magic = BA_MARK_FREE;
#endif
BA_BLOCKN(a, p, i)->next = i+2;
}
#endif
#ifdef BA_DEBUG
BA_BLOCKN(a, p, 0)->magic = BA_MARK_ALLOC;
ba_check_allocator(a, "ba_alloc after insert", __FILE__, __LINE__);
|
385483 | 2011-11-18 | Arne Goedeke | | #endif
return p->data;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | | PMOD_EXPORT void ba_free(struct block_allocator * a, void * ptr) {
|
0a3d34 | 2011-11-10 | Arne Goedeke | | uintptr_t t;
|
385483 | 2011-11-18 | Arne Goedeke | | #ifndef BA_SEGREGATE
|
0a3d34 | 2011-11-10 | Arne Goedeke | | unsigned int mask, bit;
|
385483 | 2011-11-18 | Arne Goedeke | | #endif
|
0a3d34 | 2011-11-10 | Arne Goedeke | | ba_page p;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ba_page_t n;
|
67795d | 2011-11-11 | Arne Goedeke | |
n = a->last_free;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
67795d | 2011-11-11 | Arne Goedeke | | if (n) {
|
385483 | 2011-11-18 | Arne Goedeke | | p = BA_PAGE(a, n);
|
67795d | 2011-11-11 | Arne Goedeke | | if (!BA_CHECK_PTR(a, p, ptr)) n = 0;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
|
e90c49 | 2011-11-27 | Arne Goedeke | | if (unlikely(!n)) {
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef BA_HASH_THLD
if (a->allocated > BA_HASH_THLD) {
#endif
a->last_free = n = ba_htable_lookup(a, ptr);
#ifdef BA_HASH_THLD
} else {
for (t = 1; t <= a->num_pages; t++) {
p = BA_PAGE(a, t);
if (BA_CHECK_PTR(a, p, ptr)) {
a->last_free = n = t;
break;
}
}
}
#endif
|
67795d | 2011-11-11 | Arne Goedeke | |
|
e90c49 | 2011-11-27 | Arne Goedeke | | if (unlikely(!n)) {
#ifdef BA_DEBUG
fprintf(stderr, "magnitude: %u\n", a->magnitude);
|
385483 | 2011-11-18 | Arne Goedeke | | fprintf(stderr, "did not find %p (%X[%X] | %X[%X])\n", ptr,
hash1(a, ptr), hash1(a, ptr) & BA_HASH_MASK(a),
hash2(a, ptr), hash2(a, ptr) & BA_HASH_MASK(a)
);
|
e90c49 | 2011-11-27 | Arne Goedeke | | ba_print_htable(a);
#endif
|
67795d | 2011-11-11 | Arne Goedeke | | Pike_error("Unknown pointer \n");
}
|
d2892b | 2011-12-25 | Arne Goedeke | |
#ifdef BA_DEBUG
if (n > a->num_pages) {
fprintf(stderr, "freeing from unknown page %d (num_pages: %d).\n",
n, a->num_pages);
}
#endif
|
67795d | 2011-11-11 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | | p = BA_PAGE(a, n);
|
d2892b | 2011-12-25 | Arne Goedeke | |
#ifdef BA_DEBUG
if (!BA_CHECK_PTR(a, p, ptr)) {
fprintf(stderr, "pointer %p in wrong page %p", ptr, p->data);
}
#endif
|
67795d | 2011-11-11 | Arne Goedeke | | }
|
0a3d34 | 2011-11-10 | Arne Goedeke | | t = (uintptr_t)((char*)ptr - p->data)/a->block_size;
|
385483 | 2011-11-18 | Arne Goedeke | | #ifndef BA_SEGREGATE
|
0a3d34 | 2011-11-10 | Arne Goedeke | | mask = t / (sizeof(uintptr_t) * 8);
bit = t & ((sizeof(uintptr_t)*8) - 1);
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
67795d | 2011-11-11 | Arne Goedeke | | t = p->mask[mask];
if (t & TBITMASK(uintptr_t, bit)) {
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | Pike_error("double free!");
}
|
67795d | 2011-11-11 | Arne Goedeke | | if (mask < p->free_mask)
p->free_mask = mask;
|
385483 | 2011-11-18 | Arne Goedeke | |
p->blocks_used --;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | | if (unlikely(t == 0)) {
|
e90c49 | 2011-11-27 | Arne Goedeke | | unsigned int i;
|
67795d | 2011-11-11 | Arne Goedeke | | p->free_mask = mask;
|
e90c49 | 2011-11-27 | Arne Goedeke | | for (i = 0; i < BA_MASK_NUM(a); i++) if (p->mask[i]) {
p->mask[mask] = TBITMASK(uintptr_t, bit);
return;
}
p->mask[mask] = TBITMASK(uintptr_t, bit);
|
385483 | 2011-11-18 | Arne Goedeke | | if (p->blocks_used != a->blocks-1) {
fprintf(stderr, "wrong block count detected: %u vs %u\n", p->blocks_used+1, a->blocks);
Pike_error("croak\n");
}
if (a->first == 0) {
|
6fc6ce | 2011-12-30 | Arne Goedeke | | a->first = n;
|
385483 | 2011-11-18 | Arne Goedeke | | p->next = p->prev = 0;
} else {
p->next = a->first;
BA_PAGE(a, a->first)->prev = n;
a->first = n;
}
return;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
67795d | 2011-11-11 | Arne Goedeke | | t |= TBITMASK(uintptr_t, bit);
p->mask[mask] = t;
|
a5e1b8 | 2011-11-09 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | | if (unlikely(~t == 0)) {
for (bit = 0; bit < BA_MASK_NUM(a)-1; bit++) if (~p->mask[bit]) return;
if (~(p->mask[BA_MASK_NUM(a)-1]|~BA_LAST_MASK(a))) return;
if (p->blocks_used != 0) {
fprintf(stderr, "wrong block count detected: %u vs %u\n", p->blocks_used+1, 1);
Pike_error("croak\n");
}
ba_remove_page(a, n);
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | |
} else if (p->blocks_used == 0 || p->blocks_used == a->blocks - 1) {
fprintf(stderr, "wrong number of used blocks: %u\n", p->blocks_used);
Pike_error("bad croak\n");
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|
385483 | 2011-11-18 | Arne Goedeke | | #else
if (p->blocks_used == a->blocks) {
|
67795d | 2011-11-11 | Arne Goedeke | | if (a->first == 0) {
|
6fc6ce | 2011-12-30 | Arne Goedeke | | a->first = n;
|
67795d | 2011-11-11 | Arne Goedeke | | p->next = p->prev = 0;
} else {
p->next = a->first;
BA_PAGE(a, a->first)->prev = n;
a->first = n;
}
|
385483 | 2011-11-18 | Arne Goedeke | | } else if (p->blocks_used == 1) {
|
fdfd28 | 2011-12-25 | Arne Goedeke | | if (a->empty_pages == a->max_empty_pages) {
ba_remove_page(a, n);
|
6fc6ce | 2011-12-30 | Arne Goedeke | | return;
} else a->empty_pages++;
|
385483 | 2011-11-18 | Arne Goedeke | | }
p->blocks_used --;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef BA_DEBUG
if (((ba_block_header)ptr)->magic == BA_MARK_FREE) {
fprintf(stderr, "double freed somethign\n");
}
memset(ptr, 0x75, a->block_size);
((ba_block_header)ptr)->magic = BA_MARK_FREE;
#endif
((ba_block_header)ptr)->next = p->first;
|
e90c49 | 2011-11-27 | Arne Goedeke | |
|
385483 | 2011-11-18 | Arne Goedeke | | p->first = t+1;
#endif
}
static inline void ba_remove_page(struct block_allocator * a,
|
6fc6ce | 2011-12-30 | Arne Goedeke | | ba_page_t n) {
|
385483 | 2011-11-18 | Arne Goedeke | | ba_page tmp, p;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | #ifdef BA_DEBUG
ba_check_allocator(a, "ba_remove_page", __FILE__, __LINE__);
if (a->empty_pages < a->max_empty_pages) {
Pike_error("strange things happening\n");
}
#endif
|
385483 | 2011-11-18 | Arne Goedeke | |
p = BA_PAGE(a, n);
|
475075 | 2011-11-19 | Arne Goedeke | | |
385483 | 2011-11-18 | Arne Goedeke | | fprintf(stderr, "removing page %4u\t(%p .. %p) -> %X (%X) (of %u).\n", n,
p->data, BA_LASTBLOCK(a, p), hash1(a, BA_LASTBLOCK(a,p)),
hash1(a, BA_LASTBLOCK(a,p)) & BA_HASH_MASK(a),
a->num_pages);
|
475075 | 2011-11-19 | Arne Goedeke | | */
|
385483 | 2011-11-18 | Arne Goedeke | |
|
6fc6ce | 2011-12-30 | Arne Goedeke | | IF_HASH(
ba_htable_delete(a, BA_LASTBLOCK(a, p), n);
);
|
385483 | 2011-11-18 | Arne Goedeke | |
free(p->data);
p->data = NULL;
if (a->last_free == n)
a->last_free = 0;
else if (a->last_free == a->num_pages)
a->last_free = n;
if (p->prev) {
tmp = BA_PAGE(a, p->prev);
tmp->next = p->next;
} else {
a->first = p->next;
}
if (p->next) {
tmp = BA_PAGE(a, p->next);
tmp->prev = p->prev;
}
|
de2d07 | 2011-12-25 | Arne Goedeke | | if (a->num_pages != n) {
tmp = BA_PAGE(a, a->num_pages);
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
IF_HASH(
ba_htable_delete(a, BA_LASTBLOCK(a, tmp), a->num_pages);
);
|
de2d07 | 2011-12-25 | Arne Goedeke | |
*p = *tmp;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | IF_HASH(
ba_htable_insert(a, BA_LASTBLOCK(a, tmp), n);
);
|
de2d07 | 2011-12-25 | Arne Goedeke | | if (p->next) BA_PAGE(a, p->next)->prev = n;
if (p->prev) BA_PAGE(a, p->prev)->next = n;
|
6fc6ce | 2011-12-30 | Arne Goedeke | | if (a->num_pages == a->first)
|
de2d07 | 2011-12-25 | Arne Goedeke | | a->first = n;
}
|
6fc6ce | 2011-12-30 | Arne Goedeke | | memset(BA_PAGE(a, a->num_pages), 0, sizeof(struct ba_page));
|
de2d07 | 2011-12-25 | Arne Goedeke | |
|
d2892b | 2011-12-25 | Arne Goedeke | | #ifdef BA_DEBUG
if (a->first == a->num_pages) {
fprintf(stderr, "a->first will be old removed page %d\n", a->first);
fprintf(stderr, "page %d was not moved and prev was %d\n", n, p->prev);
}
#endif
|
385483 | 2011-11-18 | Arne Goedeke | | a->num_pages--;
|
6fc6ce | 2011-12-30 | Arne Goedeke | |
#ifdef BA_DEBUG
ba_check_allocator(a, "ba_remove_page", __FILE__, __LINE__);
#endif
if (a->allocated > BA_ALLOC_INITIAL && a->num_pages < (a->allocated >> 2)) {
ba_shrink(a);
}
|
a5e1b8 | 2011-11-09 | Arne Goedeke | | }
|