pike.git / src / pike_memory.c

version» Context lines:

pike.git/src/pike_memory.c:3:   || Pike is distributed under GPL, LGPL and MPL. See the file COPYING   || for more information.   */      #include "global.h"   #include "pike_memory.h"   #include "pike_error.h"   #include "pike_macros.h"   #include "gc.h"   #include "fd_control.h" - #include "dmalloc.h" +    #include "block_allocator.h" -  + #include "pike_cpulib.h" + #include "siphash24.h"      #ifdef HAVE_SYS_MMAN_H   #include <sys/mman.h>   #endif      #ifdef HAVE_SYS_STAT_H   #include <sys/stat.h>   #endif      #ifdef HAVE_FCNTL_H   #include <fcntl.h>   #endif    - #include <stdlib.h> -  +    #include <errno.h>      int page_size;    - /* strdup() is used by several modules, so let's provide it */ - #ifndef HAVE_STRDUP - #undef strdup - char *strdup(const char *str) + long pcharp_strlen(const PCHARP a)   { -  char *res = NULL; -  if (str) { -  int len = strlen(str)+1; -  -  res = xalloc(len); -  MEMCPY(res, str, len); -  } -  return(res); - } - #endif /* !HAVE_STRDUP && !strdup */ -  - ptrdiff_t pcharp_memcmp(PCHARP a, PCHARP b, int sz) - { -  return generic_quick_binary_strcmp((char *)a.ptr, sz, a.shift, -  (char *)b.ptr, sz, b.shift); - } -  - long pcharp_strlen(PCHARP a) - { +     long len;    for(len=0;INDEX_PCHARP(a,len);len++);    return len;   }      /* NOTE: Second arg is a p_char2 to avoid warnings on some compilers. */   p_wchar1 *MEMCHR1(p_wchar1 *p, p_wchar2 c, ptrdiff_t e)   {    while(--e >= 0) if(*(p++) == (p_wchar1)c) return p-1;    return (p_wchar1 *)NULL;   }      p_wchar2 *MEMCHR2(p_wchar2 *p, p_wchar2 c, ptrdiff_t e)   {    while(--e >= 0) if(*(p++) == (p_wchar2)c) return p-1;    return (p_wchar2 *)NULL;   }    - static void swap(char *a, char *b, size_t size) - { -  size_t tmp; -  char tmpbuf[1024]; -  while(size) -  { -  tmp = MINIMUM((size_t)sizeof(tmpbuf), size); -  MEMCPY(tmpbuf,a,tmp); -  MEMCPY(a,b,tmp); -  MEMCPY(b,tmpbuf,tmp); -  size-=tmp; -  a+=tmp; -  b+=tmp; -  } - } -  - void reverse(char *memory, size_t nitems, size_t size) - { -  - #define DOSIZE(X,Y) \ -  case X: \ -  { \ -  Y tmp; \ -  Y *start=(Y *) memory; \ -  Y *end=start+nitems-1; \ -  while(start<end){tmp=*start;*(start++)=*end;*(end--)=tmp;} \ -  break; \ -  } -  - #ifdef HANDLES_UNALIGNED_MEMORY_ACCESS -  switch(size) - #else -  switch( (((size_t)memory) % size) ? 0 : size) - #endif -  { -  DOSIZE(1,B1_T) - #ifdef B2_T -  DOSIZE(2,B2_T) - #endif - #ifdef B4_T -  DOSIZE(4,B4_T) - #endif - #ifdef B16_T -  DOSIZE(8,B8_T) - #endif - #ifdef B16_T -  DOSIZE(16,B8_T) - #endif -  default: -  { -  char *start = (char *) memory; -  char *end=start+(nitems-1)*size; -  while(start<end) -  { -  swap(start,end,size); -  start+=size; -  end-=size; -  } -  } -  } - } -  +    /*    * This function may NOT change 'order'    * This function is hopefully fast enough...    */ - void reorder(char *memory, INT32 nitems, INT32 size,INT32 *order) + void reorder(char *memory, INT32 nitems, INT32 size, const INT32 *order)   { -  INT32 e; +  INT32 e, aok;    char *tmp; -  if(nitems<2) return; -  -  +  if(UNLIKELY(nitems<2)) return; +  aok = 0; +  /* +  * Prime the cache for the order array, and check the array for +  * correct ordering. At the first order mismatch, bail out and +  * start the actual reordering beyond the ordered prefix. +  * If the order turns out to be correct already, perform an early return. +  */ +  do +  if (UNLIKELY(order[aok] != aok)) +  goto unordered; +  while (LIKELY(++aok < nitems)); +  return; + unordered: +  order += aok; +  nitems -= aok;    tmp=xalloc(size * nitems); -  +  e = 0;      #undef DOSIZE -  + #undef CHECK_ALIGNED +  + #ifdef HANDLES_UNALIGNED_MEMORY_ACCESS + #define CHECK_ALIGNED(X) 0 + #else + #define CHECK_ALIGNED(X) \ +  if ((ptrdiff_t)memory & ((X)-1)) goto unaligned + #endif +    #define DOSIZE(X,Y) \ -  case X: \ +  case (X): \ +  CHECK_ALIGNED(X); \    { \    Y *from=(Y *) memory; \    Y *to=(Y *) tmp; \ -  for(e=0;e<nitems;e++) to[e]=from[order[e]]; \ +  do \ +  to[e] = from[order[e]]; \ +  while (LIKELY(++e < nitems)); \    break; \    }    -  - #ifdef HANDLES_UNALIGNED_MEMORY_ACCESS +     switch(size) - #else -  switch( (((size_t)memory) % size) ? 0 : size ) - #endif +     {    DOSIZE(1,B1_T)   #ifdef B2_T    DOSIZE(2,B2_T)   #endif   #ifdef B4_T    DOSIZE(4,B4_T)   #endif   #ifdef B8_T    DOSIZE(8,B8_T)   #endif -  + #undef CHECK_ALIGNED +  // Force aligned check for 128 bit values +  // GCC-7.2 messes up otherwise + #define CHECK_ALIGNED(X) \ +  if ((ptrdiff_t)memory & ((X)-1)) goto unaligned   #ifdef B16_T    DOSIZE(16,B16_T)   #endif       default: -  for(e=0;e<nitems;e++) MEMCPY(tmp+e*size, memory+order[e]*size, size); + unaligned: +  do +  memcpy(tmp+e*size, memory+order[e]*size, size); +  while (LIKELY(++e < nitems));    }    -  MEMCPY(memory, tmp, size * nitems); +  memcpy(memory + aok * size, tmp, size * nitems);    free(tmp);   }    - #include "siphash24.c" -  +    #if (defined(__i386__) || defined(__amd64__)) && defined(__GNUC__)   /*    * This would look much better using the compiler intrinsics, or even the    * assembler instructions directly.    *    * However, that requires a (at least in 2011) somewhat modern gcc (4.5.*).    *    * For reference:    * #define CRC32SI(H,P) H=__builtin_ia32_crc32si(H,P)    * #define CRC32SQ(H,P) H=__builtin_ia32_crc32qi(H,P)
pike.git/src/pike_memory.c:219:    __asm__ __volatile__( \    ".byte 0xf2, 0xf, 0x38, 0xf1, 0xf1;" \    :"=S"(H) :"0"(H), "c"(*(P)))      #define CRC32SQ(H,P) \    __asm__ __volatile__( \    ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" \    :"=S"(H) :"0"(H), "c"(*(P)))   #endif    - #if SIZEOF_CHAR_P == 4 - #define __cpuid(level, a, b, c, d) \ -  __asm__ ("pushl %%ebx \n\t" \ -  "cpuid \n\t" \ -  "movl %%ebx, %1 \n\t" \ -  "popl %%ebx \n\t" \ -  : "=a" (a), "=r" (b), "=c" (c), "=d" (d) \ -  : "a" (level) \ -  : "cc") - #else - #define __cpuid(level, a, b, c, d) \ -  __asm__ ("push %%rbx \n\t" \ -  "cpuid \n\t" \ -  "movl %%ebx, %1 \n\t" \ -  "pop %%rbx \n\t" \ -  : "=a" (a), "=r" (b), "=c" (c), "=d" (d) \ -  : "a" (level) \ -  : "cc") - #endif -  - #define bit_SSE4_2 (1<<20) -  - __attribute__((const)) static inline int supports_sse42( ) + ATTRIBUTE((const)) static inline int supports_sse42( )   { -  unsigned int ignore, cpuid_ecx; -  __cpuid( 0x1, ignore, ignore, cpuid_ecx, ignore ); -  return cpuid_ecx & bit_SSE4_2; +  INT32 cpuid[4]; +  x86_get_cpuid (1, cpuid); +  return cpuid[3] & bit_SSE4_2;   }      #ifdef __i386__ - __attribute__((fastcall)) + ATTRIBUTE((fastcall))   #endif   #ifdef HAVE_CRC32_INTRINSICS   /*   The intrinsics are only available if -msse4 is specified.   However, specifying that option on the command-line makes the whole runtime-test here   pointless, since gcc will use other sse4 instructions when suitable.   */ - __attribute__((target("sse4"))) + ATTRIBUTE((target("sse4")))   #endif - __attribute__((hot)) + ATTRIBUTE((hot))   static inline size_t low_hashmem_ia32_crc32( const void *s, size_t len, -  size_t nbytes, size_t key ) +  size_t nbytes, UINT64 key )   {    unsigned int h = len;    const unsigned int *p = s;       if( key )    return low_hashmem_siphash24(s,len,nbytes,key);       if( nbytes >= len )    {    /* Hash the whole memory area */
pike.git/src/pike_memory.c:286:    while( p<e ) {    CRC32SI(h, p++ );    }       len &= 3;       /* any remaining bytes. */    while( len-- )    CRC32SQ( h, c++ );    return h; +  } else if (UNLIKELY(nbytes < 32)) { +  /* Hash the whole memory area */ +  const unsigned int *e = p + (nbytes>>2); +  const unsigned char *c = (const unsigned char*)e; +  +  /* .. all full integers .. */ +  while( p<e ) { +  CRC32SI(h, p++ );    } -  else -  { +  +  nbytes &= 3; +  +  /* any remaining bytes. */ +  while( nbytes-- ) +  CRC32SQ( h, c++ ); +  return h; +  } else {    const unsigned int *e = p+(nbytes>>2);   #ifdef PIKE_DEBUG    /*    This code makes assumputions that is not true if nbytes & 3 is true.       Specifically, it will not read enough (up to 3 bytes too    little) in the first loop.       Also, if nbytes < 32 the unroll assumptions do not hold   
pike.git/src/pike_memory.c:320:    CRC32SI(h,&p[0]);    CRC32SI(h,&p[1]);    CRC32SI(h,&p[2]);    CRC32SI(h,&p[3]);    CRC32SI(h,&p[4]);    CRC32SI(h,&p[5]);    CRC32SI(h,&p[6]);    CRC32SI(h,&p[7]);    p+=8;    } - #if 0 -  while( p+3 < e ) -  { -  CRC32SI(h,&p[0]); -  CRC32SI(h,&p[1]); -  CRC32SI(h,&p[2]); -  CRC32SI(h,&p[3]); -  p+=4; -  } -  while( p+1 < e ) -  { -  CRC32SI(h,&p[0]); -  CRC32SI(h,&p[1]); -  p+=2; -  } -  while( p<e ) { -  CRC32SI(h,p++); -  } - #endif +     /* include 8 bytes from the end. Note that this might be a    * duplicate of the previous bytes.    *    * Also note that this means we are rather likely to read    * unaligned memory. That is OK, however.    */    e = (const unsigned int *)((const unsigned char *)s+len-8);    CRC32SI(h,e++);    CRC32SI(h,e);    }   #if SIZEOF_CHAR_P > 4    /* FIXME: We could use the long long crc32 instructions that work on 64 bit values.    * however, those are only available when compiling to amd64.    */    return (((size_t)h)<<32) | h; - #endif + #else    return h; -  + #endif   }      #ifdef __i386__ - __attribute__((fastcall)) + ATTRIBUTE((fastcall))   #endif -  size_t (*low_hashmem)(const void *, size_t, size_t, size_t); +  size_t (*low_hashmem)(const void *, size_t, size_t, UINT64);      static void init_hashmem()   {    if( supports_sse42() )    low_hashmem = low_hashmem_ia32_crc32;    else    low_hashmem = low_hashmem_siphash24;   }   #else   static void init_hashmem(){}      ATTRIBUTE((hot)) -  size_t low_hashmem(const void *a, size_t len_, size_t mlen_, size_t key_) +  size_t low_hashmem(const void *a, size_t len_, size_t mlen_, UINT64 key_)   {    return low_hashmem_siphash24(a, len_, mlen_, key_);   }   #endif      ATTRIBUTE((hot))    size_t hashmem(const void *a, size_t len_, size_t mlen_)   {    return low_hashmem(a, len_, mlen_, 0);   }
pike.git/src/pike_memory.c:411:    free(mem);   }      PMOD_EXPORT void *debug_xmalloc(size_t s)   {    return malloc(s);   }      PMOD_EXPORT void *debug_xrealloc(void *m, size_t s)   { -  return realloc(m,s); +  void *ret = realloc(m,s); +  if (ret) return ret; +  Pike_error(msg_out_of_mem_2, s); +  return NULL;   }      PMOD_EXPORT void *debug_xcalloc(size_t n, size_t s)   {    void *ret;    if(!n || !s)    Pike_error("Allocating zero bytes.\n");       ret=(void *)calloc(n, s);    if(ret) return ret;       Pike_error(msg_out_of_mem_2, n*s);    return 0;   }    - PMOD_EXPORT void *aligned_alloc(size_t size, size_t alignment) { + PMOD_EXPORT void *xalloc_aligned(size_t size, size_t alignment) {    void * ret;       if (!size) return 0;      #ifdef HAVE_POSIX_MEMALIGN    if (posix_memalign(&ret, alignment, size)) {    Pike_error(msg_out_of_mem_2, size);    }   #else    if ((ret = memalign(alignment, size)) == NULL) {
pike.git/src/pike_memory.c:450:   #endif       return ret;   }      PMOD_EXPORT char *debug_xstrdup(const char *src)   {    char *dst = NULL;    if (src) {    int len = strlen (src) + 1; -  dst = malloc (len); -  MEMCPY (dst, src, len); +  dst = xalloc (len); +  memcpy (dst, src, len);    }    return dst;   }    - char *debug_qalloc(size_t size) - { -  char *ret; -  if(!size) return 0; -  -  ret=(char *)malloc(size); -  if(ret) return ret; -  - #ifdef SOFTLIM -  { -  struct rlim lim; -  if(getrlimit(RLIMIT_DATA,&lim)>= 0) -  { -  if(lim.rlim_cur < lim.rlim_max) -  { -  lim.rlim_cur+=size; -  while(1) -  { -  softlim_should_be=lim.rlim_cur; -  if(lim.rlim_cur > lim.rlim_max) -  lim.rlim_cur=lim.rlim_max; -  -  if(setrlimit(RLIM_DATA, &lim)>=0) -  { -  ret=(char *)malloc(size); -  if(ret) return ret; -  } -  if(lim.rlim_cur >= lim.rlim_max) break; -  lim.rlim_cur+=4096; -  } -  } -  } -  } - #endif -  -  Pike_fatal("Completely out of memory - " -  "failed to allocate %"PRINTSIZET"d bytes!\n", size); -  /* NOT_REACHED */ -  return NULL; /* Keep the compiler happy. */ - } -  +    /*    * mexec_*()    *    * Allocation of executable memory.    */      #ifdef MEXEC_USES_MMAP      #define MEXEC_ALLOC_CHUNK_SIZE page_size   
pike.git/src/pike_memory.c:520: Inside #if defined(MAP_ANONYMOUS)
     #ifdef MAP_ANONYMOUS   /* Note: mmap ANON fails (EINVAL) on Solaris if the fd isn't -1. */   #define dev_zero -1   #else   static int dev_zero = -1;   #define INIT_DEV_ZERO   #define MAP_ANONYMOUS 0   #endif    - static INLINE void *mexec_do_alloc (void *start, size_t length) + static inline void *mexec_do_alloc (void *start, size_t length)   {    void *blk = mmap(start, length, PROT_EXEC|PROT_READ|PROT_WRITE,    MAP_PRIVATE|MAP_ANONYMOUS, dev_zero, 0);    if (blk == MAP_FAILED) {    fprintf(stderr, "mmap of %"PRINTSIZET"u bytes failed, errno=%d. "    "(dev_zero=%d)\n", length, errno, dev_zero);    return NULL;    }    return blk;   }      #elif defined (_WIN32)      /* VirtualAlloc practically never succeeds to give us a new block    * adjacent to an earlier allocated block if we use the page size.    * Some testing shows that VirtualAlloc often pads alloc requests to    * 64kb pages (at least on Windows Server 2003), so we use that as the    * basic allocation unit instead of the page size. */   #define MEXEC_ALLOC_CHUNK_SIZE (64 * 1024)    - static INLINE void *mexec_do_alloc (void *start, size_t length) + static inline void *mexec_do_alloc (void *start, size_t length)   {    void *blk = VirtualAlloc (start, length, MEM_RESERVE|MEM_COMMIT,    PAGE_EXECUTE_READWRITE);    if (!blk) {    blk = VirtualAlloc (NULL, length, MEM_RESERVE|MEM_COMMIT,    PAGE_EXECUTE_READWRITE);    if (!blk) {    fprintf (stderr, "VirtualAlloc of %"PRINTSIZET"u bytes failed. "    "Error code: %d\n", length, GetLastError());    return NULL;
pike.git/src/pike_memory.c:567:   #endif /* _WIN32 */      #ifdef USE_MY_MEXEC_ALLOC      /* #define MY_MEXEC_ALLOC_STATS */      #ifdef MY_MEXEC_ALLOC_STATS   static size_t sep_allocs = 0, grow_allocs = 0, total_size = 0;   #endif    - #if 0 - #define MEXEC_MAGIC 0xdeadfeedf00dfaddLL - #endif /* 0 */ +    struct mexec_block {    struct mexec_hdr *hdr;    ptrdiff_t size;   #ifdef MEXEC_MAGIC    unsigned long long magic;   #endif /* MEXEC_MAGIC */   };   struct mexec_free_block {    struct mexec_free_block *next;    ptrdiff_t size;
pike.git/src/pike_memory.c:815:    hdr->free = blk;    }    } else {    prev->next = blk;    }    } else {    if ((((char *)blk) + blk->size) == hdr->bottom) {    /* Join with bottom. */    hdr->bottom = (char *)blk;    hdr->free = blk->next; - #if 0 -  if (hdr->bottom == (char *)(hdr + 1)) { -  /* The entire mmapped block is free. -  * FIXME: Consider unmapping it. -  */ -  } - #endif /* 0 */ +     } else {    hdr->free = blk;    }    }    verify_mexec_hdr(hdr);   }      void *mexec_alloc(size_t sz)   {    struct mexec_hdr *hdr;
pike.git/src/pike_memory.c:1010:    size_t oldsz = oldblk[-1];    size_t *newblk = realloc (oldblk - 1, sz + sizeof (size_t));    if (!newblk) return NULL;    if (newblk != oldblk)    VALGRIND_DISCARD_TRANSLATIONS (oldblk, oldsz);    else if (oldsz > sz)    VALGRIND_DISCARD_TRANSLATIONS (newblk + sz, oldsz - sz);    *newblk = sz;    return newblk + 1;    } -  return mexec_malloc (sz); +  return mexec_alloc (sz);   }      void mexec_free (void *ptr)   {    size_t *blk = ptr;    VALGRIND_DISCARD_TRANSLATIONS (blk, blk[-1]);    free (blk - 1);   }      #else /* !USE_MY_MEXEC_ALLOC && !VALGRIND_DISCARD_TRANSLATIONS */
pike.git/src/pike_memory.c:1093:      #undef DO_IF_DMALLOC   #define DO_IF_DMALLOC(X)   #undef DO_IF_NOT_DMALLOC   #define DO_IF_NOT_DMALLOC(X) X         #include "threads.h"      #ifdef _REENTRANT - static MUTEX_T debug_malloc_mutex; + static PIKE_MUTEX_T debug_malloc_mutex;   #endif         #undef malloc   #undef free   #undef realloc   #undef calloc   #undef strdup   #undef main   
pike.git/src/pike_memory.c:1309:    LOWDEBUG3("realloc <-",x);    LOWDEBUG3("realloc ",y);       if(FAKEMALLOCED(x) || !real_realloc)    {    old_size = x?BASEOF(x,fakemallocblock,body.block)->size :0;    LOWDEBUG3("realloc oldsize",old_size);    if(old_size >= y) return x;    ret=malloc(y);    if(!ret) return 0; -  MEMCPY(ret, x, old_size); +  memcpy(ret, x, old_size);    if(x) free(x);    }else{    ret=debug_realloc(x, y, DMALLOC_LOCATION());    }   #ifndef REPORT_ENCAPSULATED_MALLOC    dmalloc_accept_leak(ret);   #endif    LOWDEBUG3("realloc --> ",ret);    return ret;   }
pike.git/src/pike_memory.c:1336:       LOWDEBUG3("fake_realloc <-",x);    LOWDEBUG3("fake_realloc ",y);       if(FAKEMALLOCED(x) || !real_realloc)    {    old_size = x?BASEOF(x,fakemallocblock,body.block)->size :0;    if(old_size >= y) return x;    ret=malloc(y);    if(!ret) return 0; -  MEMCPY(ret, x, old_size); +  memcpy(ret, x, old_size);    if(x) free(x);    }else{    ret=real_realloc(x,y);    }   #ifndef REPORT_ENCAPSULATED_MALLOC    dmalloc_accept_leak(ret);   #endif    LOWDEBUG3("fake_realloc --> ",ret);    return ret;   }      void *calloc(size_t x, size_t y)   {    void *ret;    LOWDEBUG3("calloc x",x);    LOWDEBUG3("calloc y",y);    ret=malloc(x*y); -  if(ret) MEMSET(ret,0,x*y); +  if(ret) memset(ret,0,x*y);   #ifndef REPORT_ENCAPSULATED_MALLOC    dmalloc_accept_leak(ret);   #endif    LOWDEBUG3("calloc --> ",ret);    return ret;   }      void *fake_calloc(size_t x, size_t y)   {    void *ret;    ret=fake_malloc(x*y); -  if(ret) MEMSET(ret,0,x*y); +  if(ret) memset(ret,0,x*y);   #ifndef REPORT_ENCAPSULATED_MALLOC    dmalloc_accept_leak(ret);   #endif    LOWDEBUG3("fake_calloc --> ",ret);    return ret;   }         #define DMALLOC_USING_DLOPEN   
pike.git/src/pike_memory.c:1553: Inside #if DEBUG_MALLOC_PAD - 0 > 0
   PIKE_MEM_NA_RANGE(mem + size, DEBUG_MALLOC_PAD);    }       else {    unsigned long q,e;    q= (((long)mem) ^ 0x555555) + (size * 9248339);       /* fprintf(stderr,"Padding %p(%d) %ld\n",mem, size, q); */   #if 1    q%=RNDSIZE; -  MEMCPY(mem - DEBUG_MALLOC_PAD, rndbuf+q, DEBUG_MALLOC_PAD); -  MEMCPY(mem + size, rndbuf+q, DEBUG_MALLOC_PAD); +  memcpy(mem - DEBUG_MALLOC_PAD, rndbuf+q, DEBUG_MALLOC_PAD); +  memcpy(mem + size, rndbuf+q, DEBUG_MALLOC_PAD);   #else    for(e=0;e< DEBUG_MALLOC_PAD; e+=4)    {    char tmp;    q=(q<<13) ^ ~(q>>5);      #define BLORG(X,Y) \    tmp=(Y); \    mem[e+(X)-DEBUG_MALLOC_PAD] = tmp; \    mem[size+e+(X)] = tmp;
pike.git/src/pike_memory.c:1607:    }else{    size = ~size;    }    }       if (PIKE_MEM_CHECKER()) return;      /* fprintf(stderr,"Checking %p(%d) %ld\n",mem, size, q); */   #if 1    /* optimization? */ -  if(MEMCMP(mem - DEBUG_MALLOC_PAD, mem+size, DEBUG_MALLOC_PAD)) +  if(memcmp(mem - DEBUG_MALLOC_PAD, mem+size, DEBUG_MALLOC_PAD))    {    q= (((long)mem) ^ 0x555555) + (size * 9248339);       q%=RNDSIZE; -  if(MEMCMP(mem - DEBUG_MALLOC_PAD, rndbuf+q, DEBUG_MALLOC_PAD)) +  if(memcmp(mem - DEBUG_MALLOC_PAD, rndbuf+q, DEBUG_MALLOC_PAD))    {    out_biking=1;    fprintf(stderr,"Pre-padding overwritten for "    "block at %p (size %ld)!\n", mem, size);    describe(mem);    abort();    }    -  if(MEMCMP(mem + size, rndbuf+q, DEBUG_MALLOC_PAD)) +  if(memcmp(mem + size, rndbuf+q, DEBUG_MALLOC_PAD))    {    out_biking=1;    fprintf(stderr,"Post-padding overwritten for "    "block at %p (size %ld)!\n", mem, size);    describe(mem);    abort();    }       out_biking=1;    fprintf(stderr,"Padding completely screwed up for "
pike.git/src/pike_memory.c:1679:    BLORG(3, (q >> 15) | 1)    }   #endif   }   #else   #define do_pad(X,Y) (X)   #define check_pad(M,X)   #endif       - static INLINE unsigned long lhash(struct memhdr *m, LOCATION location) + static inline unsigned long lhash(struct memhdr *m, LOCATION location)   {    unsigned long l;   #ifdef DMALLOC_USE_HASHBASE    l = m->hashbase;   #else /* !DMALLOC_USE_HASHBASE */    l=(long)m;    l*=53;   #endif /* DMALLOC_USE_HASHBASE */    l+=(long)location;    l%=LHSIZE;
pike.git/src/pike_memory.c:1883: Inside #if defined(DMALLOC_TRACE_MEMLOC)
   mlhash[l]=ml;   #ifdef DMALLOC_TRACE_MEMLOC    if (ml == DMALLOC_TRACE_MEMLOC) {    fprintf(stderr, "add_loc: Allocated memloc %p location %s memhdr: %p data: %p\n",    ml, ml->location, ml->mh, ml->mh->data);    }   #endif /* DMALLOC_TRACE_MEMLOC */    return;   }    - static void remove_location(struct memhdr *mh, LOCATION location) - { -  struct memloc *ml,**prev; -  unsigned long l; + PMOD_EXPORT LOCATION dmalloc_default_location=0;    - #if defined(DMALLOC_VERIFY_INTERNALS) && !defined(__NT__) && defined(PIKE_THREADS) -  if(!mt_trylock(& debug_malloc_mutex)) -  Pike_fatal("remove_location running unlocked!\n"); - #endif -  - #if DEBUG_MALLOC - 0 < 2 -  if(find_location(&no_leak_memlocs, location)) return; - #endif -  -  if (!(ml = find_location(mh, location))) { -  return; -  } -  -  /* Unlink from the hashtable. */ -  if ((ml->hash_prev[0] = ml->hash_next)) { -  ml->hash_next->hash_prev = ml->hash_prev; -  } -  -  prev=&mh->locations; -  while((ml=*prev)) -  { - #ifdef DMALLOC_VERIFY_INTERNALS -  if (ml->mh != mh) { -  Pike_fatal("Non-owned memloc in location list!\n"); -  } - #endif -  -  if(ml->location==location) -  { -  *prev=ml->next; - #ifdef DMALLOC_VERIFY_INTERNALS -  mh->times -= ml->times; - #endif - #ifdef DMALLOC_TRACE_MEMLOC -  if (ml == DMALLOC_TRACE_MEMLOC) { -  fprintf(stderr, "rem_loc: Freeing memloc %p location %s memhdr: %p data: %p\n", -  ml, ml->location, ml->mh, ml->mh->data); -  } - #endif /* DMALLOC_TRACE_MEMLOC */ -  really_free_memloc(ml); -  return; -  }else{ -  prev=&ml->next; -  } -  } - } -  - LOCATION dmalloc_default_location=0; -  +    static struct memhdr *low_make_memhdr(void *p, int s, LOCATION location   #ifdef DMALLOC_C_STACK_TRACE    , c_stack_frame *bt, int bt_len   #endif    )   {    struct memhdr *mh = get_memhdr(p);    struct memloc *ml = alloc_memloc();    unsigned long l;   
pike.git/src/pike_memory.c:1964:    mh->size=s;    mh->flags=0;   #ifdef DMALLOC_VERIFY_INTERNALS    mh->times=1;   #endif    ml->next = mh->locations;    mh->locations = ml;    mh->gc_generation=gc_generation * 1000 + Pike_in_gc;   #ifdef DMALLOC_C_STACK_TRACE    if (bt_len > 0) { -  MEMCPY (mh->alloc_bt, bt, bt_len * sizeof (c_stack_frame)); +  memcpy (mh->alloc_bt, bt, bt_len * sizeof (c_stack_frame));    mh->alloc_bt_len = bt_len;    }    else    mh->alloc_bt_len = 0;   #endif    ml->location=location;    ml->times=1;    ml->mh=mh;    ml->hash_prev = &mlhash[l];    if ((ml->hash_next = mlhash[l])) {
pike.git/src/pike_memory.c:2080:      PMOD_EXPORT int dmalloc_unregister(void *p, int already_gone)   {    int ret;    mt_lock(&debug_malloc_mutex);    ret=low_dmalloc_unregister(p,already_gone);    mt_unlock(&debug_malloc_mutex);    return ret;   }    - static int low_dmalloc_mark_as_free(void *p, int already_gone) + static int low_dmalloc_mark_as_free(void *p, int UNUSED(already_gone))   {    struct memhdr *mh=find_memhdr(p);    if(mh)    {    if (mh->flags & MEM_TRACE) {    fprintf(stderr, "Marking memhdr %p as free\n", mh);    }    if(!(mh->flags & MEM_FREE))    {    mh->size=~mh->size;
pike.git/src/pike_memory.c:2175:    flush_blocks_to_free();    if ((m=(char *)real_malloc(s + DEBUG_MALLOC_PAD*2))) {    GET_ALLOC_BT (bt);    m=do_pad(m, s);    low_make_memhdr(m, s, location BT_ARGS (bt))->flags|=MEM_PADDED;    }    }       if(verbose_debug_malloc)    fprintf(stderr, "malloc(%ld) => %p (%s)\n", -  DO_NOT_WARN((long)s), -  m, LOCATION_NAME(location)); +  (long)s, m, LOCATION_NAME(location));       mt_unlock(&debug_malloc_mutex);    return m;   }      PMOD_EXPORT void *debug_calloc(size_t a, size_t b, LOCATION location)   {    void *m=debug_malloc(a*b,location);    if(m) -  MEMSET(m, 0, a*b); +  memset(m, 0, a*b);       if(verbose_debug_malloc)    fprintf(stderr, "calloc(%ld, %ld) => %p (%s)\n", -  DO_NOT_WARN((long)a), -  DO_NOT_WARN((long)b), -  m, LOCATION_NAME(location)); +  (long)a, (long)b, m, LOCATION_NAME(location));       return m;   }      PMOD_EXPORT void *debug_realloc(void *p, size_t s, LOCATION location)   {    char *m,*base;    struct memhdr *mh = 0;    mt_lock(&debug_malloc_mutex);   
pike.git/src/pike_memory.c:2225:    add_location(mh, location);    move_memhdr(mh, m);    }    else {    GET_ALLOC_BT (bt);    low_make_memhdr(m, s, location BT_ARGS (bt))->flags|=MEM_PADDED;    }    }    if(verbose_debug_malloc)    fprintf(stderr, "realloc(%p, %ld) => %p (%s)\n", -  p, -  DO_NOT_WARN((long)s), -  m, LOCATION_NAME(location)); +  p, (long)s, m, LOCATION_NAME(location));    mt_unlock(&debug_malloc_mutex);    return m;   }      PMOD_EXPORT void debug_free(void *p, LOCATION location, int mustfind)   {    struct memhdr *mh;    if(!p) return;    mt_lock(&debug_malloc_mutex);   
pike.git/src/pike_memory.c:2259:    fprintf(stderr,"Lost track of a mustfind memory block: %p!\n",p);    abort();    }       if(!exiting && mh)    {    void *p2;    if (PIKE_MEM_CHECKER())    PIKE_MEM_NA_RANGE(p, mh->size);    else -  MEMSET(p, 0x55, mh->size); +  memset(p, 0x55, mh->size);    if(mh->size < MAX_UNFREE_MEM/FREE_DELAY)    {    add_location(mh, location);    mh->size = ~mh->size;    mh->flags|=MEM_FREE | MEM_IGNORE_LEAK;    blocks_to_free_ptr++;    blocks_to_free_ptr%=FREE_DELAY;    p2=blocks_to_free[blocks_to_free_ptr];    blocks_to_free[blocks_to_free_ptr]=p;    if((p=p2))
pike.git/src/pike_memory.c:2317:    struct memhdr *mh;    mt_lock(&debug_malloc_mutex);    mh=my_find_memhdr(p,0);    res = mh && mh->size>=0;    if (res && must_be_freed)    res = !(mh->flags & MEM_IGNORE_LEAK);    mt_unlock(&debug_malloc_mutex);    return res;   }    - #if 0 - /* Disabled since it isn't used. */ - void dmalloc_check_block_free(void *p, LOCATION location, -  char *struct_name, describe_block_fn *describer) - { -  struct memhdr *mh; -  mt_lock(&debug_malloc_mutex); -  mh=my_find_memhdr(p,0); -  -  if(mh && mh->size>=0) -  { -  if(!(mh->flags & MEM_IGNORE_LEAK)) -  { -  fprintf(stderr, "Freeing %s still in use %p at %s.\n", -  struct_name ? struct_name : "small block", p, LOCATION_NAME(location)); -  if (describer) describer (p); -  debug_malloc_dump_references(p,0,2,0); -  } -  mh->flags |= MEM_FREE | MEM_IGNORE_LEAK; -  mh->size = ~mh->size; -  } -  -  mt_unlock(&debug_malloc_mutex); - } - #endif -  +    PMOD_EXPORT void dmalloc_free(void *p)   {    debug_free(p, DMALLOC_LOCATION(), 0);   }      PMOD_EXPORT char *debug_strdup(const char *s, LOCATION location)   {    char *m;    long length;    length=strlen(s);    m=(char *)debug_malloc(length+1,location); -  MEMCPY(m,s,length+1); +  memcpy(m,s,length+1);       if(verbose_debug_malloc)    fprintf(stderr, "strdup(\"%s\") => %p (%s)\n", s, m, LOCATION_NAME(location));       return m;   }      #ifdef WRAP   void *__wrap_malloc(size_t size)   {
pike.git/src/pike_memory.c:2402:    size_t file_len;    INT_TYPE line;    const char *extra;   };      static void parse_location (struct memloc *l, struct parsed_location *pl)   {    const char *p;    pl->file = LOCATION_NAME (l->location);    -  p = STRCHR (pl->file, ' '); +  p = strchr (pl->file, ' ');    if (p)    pl->extra = p;    else -  pl->extra = STRCHR (pl->file, 0); +  pl->extra = strchr (pl->file, 0);    -  p = STRCHR (pl->file, ':'); +  p = strchr (pl->file, ':');    if (p && p < pl->extra) {    const char *pp; -  while ((pp = STRCHR (p + 1, ':')) && pp < pl->extra) p = pp; -  pl->line = STRTOL (p + 1, NULL, 10); +  while ((pp = strchr (p + 1, ':')) && pp < pl->extra) p = pp; +  pl->line = strtol (p + 1, NULL, 10);    pl->file_len = p - pl->file;    }    else {    pl->line = -1;    pl->file_len = pl->extra - pl->file;    }   }      static void sort_locations (struct memhdr *hdr)   {
pike.git/src/pike_memory.c:2654:    if(!(flags & 2))    find_references_to(x,indent+2,depth-1,flags);    }    else    {    fprintf(stderr,"%*s<<<=- No known references to this block -=>>>\n",indent,"");    }    }   }    - void list_open_fds(void) + PMOD_EXPORT void list_open_fds(void)   {    unsigned long h;    mt_lock(&debug_malloc_mutex);       for(h=0;h<(unsigned long)memhdr_hash_table_size;h++)    {    struct memhdr *m;    for(m=memhdr_hash_table[h];m;m=m->next)    {    struct memhdr *tmp;
pike.git/src/pike_memory.c:2717:    continue;    }    m->flags |= MEM_SCANNED;       counter++;    if (counter > num_memhdr) {    Pike_fatal("Found too many memhdrs!\n");    }    if( ! ((sizeof(void *)-1) & (size_t)p ))    { - #if 0 -  if(m->size > 0) -  { - #if defined(__NT__) && !defined(__GNUC__) -  __try { - #endif - #ifdef PIKE_EXTRA_DEBUG -  fprintf(stderr, "PIKE_EXTRA_DEBUG:\n" -  " Scanning memory block at %p, %ld bytes, " -  "generation %d, flags: 0x%08x\n", -  m->data, m->size, m->gc_generation, m->flags); - #endif /* PIKE_EXTRA_DEBUG */ -  for(e=0;e<m->size/sizeof(void *);e++) { -  void *addr = p[e]; -  if (!addr || ((sizeof(void *)-1) & (size_t)addr)) { -  /* No need to hunt for memhdrs for NULL or -  * unaligned addresses. -  * This also filters out memory areas that have -  * been cleared by block_alloc.h:really_free_*() et al. -  * /grubba 2003-03-15 -  */ -  continue; +     } -  /* NOTE: We must not use find_memhdr() here, -  * since it might alter the data-structure -  * we're looping over... -  * -  * A specific case is a bucket containing two -  * self-referring memhdrs, using find_memhdr() -  * will then cause an infinite loop. -  * /grubba 2003-03-16 -  */ -  if((tmp=just_find_memhdr(addr))) -  tmp->flags |= MEM_REFERENCED; +     } - #if defined(__NT__) && !defined(__GNUC__) +     } -  __except( 1 ) { -  fprintf(stderr,"*** DMALLOC memory access error ***\n"); -  fprintf(stderr,"Failed to access this memory block:\n"); -  fprintf(stderr,"Block: %p, size=%ld, gc_generation=%d, flags=%d\n", -  m->data, -  m->size, -  m->gc_generation, -  m->flags); -  dump_memhdr_locations(m, 0, 0); -  fprintf(stderr,"-----------------------------------\n"); -  } - #endif -  } - #endif -  } -  } -  } +        memheader_references_located=1;   }      void search_all_memheaders_for_references(void)   {    mt_lock(&debug_malloc_mutex);    low_search_all_memheaders_for_references();    mt_unlock(&debug_malloc_mutex);   }
pike.git/src/pike_memory.c:2907: Inside #if defined(DMALLOC_PROFILE)
   fprintf(stderr,"memhdrs, hashed %ld, free %ld, ignored %ld, other %ld\n",    total_memhdr,    free_memhdr,    ignored_memhdr,    total_memhdr - free_memhdr - ignored_memhdr);    }   #endif       }    mt_unlock(&debug_malloc_mutex); -  mt_destroy(&debug_malloc_mutex); +    }      #ifdef _REENTRANT   static void lock_da_lock(void)   {    mt_lock(&debug_malloc_mutex);   }      static void unlock_da_lock(void)   {
pike.git/src/pike_memory.c:2933:   {    long e;    static int initialized=0;    if(!initialized)    {    initialized=1;   #ifdef DMALLOC_REMEMBER_LAST_LOCATION    th_key_create(&dmalloc_last_seen_location, 0);   #endif    init_memhdr_hash(); -  MEMSET(mlhash, 0, sizeof(mlhash)); +  memset(mlhash, 0, sizeof(mlhash));       for(e=0;e<(long)NELEM(rndbuf);e++) rndbuf[e]= (rand() % 511) | 1;      #if DEBUG_MALLOC_PAD & 3    fprintf(stderr,"DEBUG_MALLOC_PAD not dividable by four!\n");    exit(99);   #endif      #ifdef _REENTRANT   #ifdef mt_init_recursive
pike.git/src/pike_memory.c:3023:       if (bin_data) hval += hashmem (bin_data, bin_data_len, 64);       h=hval % DSTRHSIZE;       mt_lock(&debug_malloc_mutex);       for(prev = dstrhash + h; (str=*prev); prev = &str->next)    {    if(hval == str->hval && -  !STRNCMP(str->str+1, file, len) && +  !strncmp(str->str+1, file, len) &&    str->str[len+1]==':' &&    LOCATION_TYPE (str->str) == type && -  STRTOL(str->str+len+2, NULL, 10) == line) +  strtol(str->str+len+2, NULL, 10) == line)    {       if (name) { -  char *s = STRCHR (str->str + len + 2, ' '); +  char *s = strchr (str->str + len + 2, ' ');    if (!s) continue;    s++;    if (strcmp (s, name)) continue;    }       if (bin_data) {    /* Can assume that str also carries a bin_data blob since it    * has the same type. */    unsigned char *str_bin_base =    (unsigned char *) str->str + len + strlen (str->str + len) + 1; -  unsigned int str_bin_len = EXTRACT_UWORD (str_bin_base); +  unsigned int str_bin_len = get_unaligned16 (str_bin_base);    str_bin_base += 2;    if (str_bin_len != bin_data_len || -  MEMCMP (bin_data, str_bin_base, str_bin_len)) +  memcmp (bin_data, str_bin_base, str_bin_len))    continue;    }       *prev=str->next;    str->next=dstrhash[h];    dstrhash[h]=str;    break;    }    }   
pike.git/src/pike_memory.c:3077:    sprintf(str->str, "%c%s:%s %s", type, file, line_str, name);    else    sprintf(str->str, "%c%s:%s", type, file, line_str);       if (bin_data) {    unsigned INT16 bl = (unsigned INT16) bin_data_len;    if (bl != bin_data_len)    Pike_fatal ("Too long bin_data blob: %u\n", bin_data_len);    ((unsigned char *) str->str)[l + 1] = ((unsigned char *) &bl)[0];    ((unsigned char *) str->str)[l + 2] = ((unsigned char *) &bl)[1]; -  MEMCPY (str->str + l + 3, bin_data, bin_data_len); +  memcpy (str->str + l + 3, bin_data, bin_data_len);    }       str->hval=hval;    str->next=dstrhash[h];    dstrhash[h]=str;    }       mt_unlock(&debug_malloc_mutex);       return str->str;   }    - LOCATION dynamic_location(const char *file, INT_TYPE line) + PMOD_EXPORT LOCATION dynamic_location(const char *file, INT_TYPE line)   {    return low_dynamic_location('D',file,line, NULL, NULL, 0);   }         PMOD_EXPORT void * debug_malloc_name(void *p,const char *file, INT_TYPE line)   {    if(p)    {    struct memhdr *mh;
pike.git/src/pike_memory.c:3144:    names++;    }    }    }       mt_unlock(&debug_malloc_mutex);    }    return names;   }    - char *dmalloc_find_name(void *p) + const char *dmalloc_find_name(void *p)   { -  char *name=0; +  const char *name=0;    if(p)    {    struct memhdr *mh;    mt_lock(&debug_malloc_mutex);       if((mh=my_find_memhdr(p,0)))    {    struct memloc *l;    for(l=mh->locations;l;l=l->next)    {
pike.git/src/pike_memory.c:3189: Inside #if defined(DMALLOC_C_STACK_TRACE)
   l = low_dynamic_location ('B', file, line, name,    /* Shave off one entry for our own frame. */    (unsigned char *) (bt + 1),    (n - 1) * sizeof (c_stack_frame));    else   #endif    l = low_dynamic_location ('D', file, line, name, NULL, 0);    return debug_malloc_update_location (p, l);   }    + #ifdef DMALLOC_C_STACK_TRACE   static void dump_location_bt (LOCATION location, int indent, const char *prefix)   { - #ifdef DMALLOC_C_STACK_TRACE +     if (LOCATION_TYPE (location) == 'B') {    c_stack_frame bt[BT_MAX_FRAMES];    int i, frames;    unsigned char *bin_base =    (unsigned char *) location + strlen (location) + 1; -  unsigned int bin_len = EXTRACT_UWORD (bin_base); -  MEMCPY ((unsigned char *) bt, bin_base + 2, bin_len); +  unsigned int bin_len = get_unaligned16 (bin_base); +  memcpy (bt, bin_base + 2, bin_len);    frames = bin_len / sizeof (c_stack_frame);       for (i = 0; i < frames; i++) {    fprintf (stderr, "%*s%s", indent, "", prefix);    backtrace_symbols_fd (bt + i, 1, 2);    }    } - #endif +    } -  + #else + static void dump_location_bt (LOCATION UNUSED(location), int UNUSED(indent), +  const char *UNUSED(prefix)) + { } + #endif      PMOD_EXPORT int debug_malloc_touch_fd(int fd, LOCATION location)   {    if(fd==-1) return fd;    debug_malloc_update_location( FD2PTR(fd), location);    return fd;   }      PMOD_EXPORT int debug_malloc_register_fd(int fd, LOCATION location)   {
pike.git/src/pike_memory.c:3229:    dmalloc_unregister( FD2PTR(fd), 1);    dmalloc_register( FD2PTR(fd), 0, location);    return fd;   }      PMOD_EXPORT void debug_malloc_accept_leak_fd(int fd)   {    dmalloc_accept_leak(FD2PTR(fd));   }    - PMOD_EXPORT int debug_malloc_close_fd(int fd, LOCATION location) + PMOD_EXPORT int debug_malloc_close_fd(int fd, LOCATION UNUSED(location))   {    if(fd==-1) return fd;   #ifdef DMALLOC_TRACK_FREE    dmalloc_mark_as_free( FD2PTR(fd), 1 );   #else /* !DMALLOC_TRACK_FREE */    remove_memhdr(FD2PTR(fd));   #endif /* DMALLOC_TRACK_FREE */    return fd;   }   
pike.git/src/pike_memory.c:3289:    int recur_offset;    struct memory_map *recur;   };      static struct block_allocator memory_map_allocator = BA_INIT_PAGES(sizeof(struct memory_map), 8);      static struct memory_map * alloc_memory_map() {    return ba_alloc(&memory_map_allocator);   }    - static void really_free_memory_map(struct memory_map * m) { -  ba_free(&memory_map_allocator, m); - } -  +    void count_memory_in_memory_maps(size_t * n, size_t * s) {    ba_count_all(&memory_map_allocator, n, s);   }      static struct block_allocator memory_map_entry_allocator    = BA_INIT_PAGES(sizeof(struct memory_map_entry), 16);      static struct memory_map_entry * alloc_memory_map_entry() {    return ba_alloc(&memory_map_entry_allocator);   }    - static void really_free_memory_map_entry(struct memory_map_entry * m) { -  ba_free(&memory_map_entry_allocator, m); - } -  +    void count_memory_in_memory_map_entrys(size_t * n, size_t * s) {    ba_count_all(&memory_map_entry_allocator, n, s);   }      void dmalloc_set_mmap(void *ptr, struct memory_map *m)   {    m->refs++;    debug_malloc_update_location(ptr, m->name+1);   }   
pike.git/src/pike_memory.c:3469:    exit_memhdr_hash();       for (i = 0; i < DSTRHSIZE; i++) {    struct dmalloc_string *str = dstrhash[i], *next;    while (str) {    next = str->next;    free(str);    str = next;    }    } +  +  mt_destroy(&debug_malloc_mutex);   }      #endif /* DEBUG_MALLOC */    -  + #if defined(__GNUC__) && defined(__SSE__) && defined(HAVE_EMMINTRIN_H) + #include <emmintrin.h> + #define SSE2 + #endif +  + static inline void low_zero(void *p, size_t n) + { +  volatile char * _p = (char *)p; +  while (n--) *_p++ = 0; + } +  + PMOD_EXPORT void secure_zero(void *p, size_t n) + { + #ifdef SSE2 +  if( n > 256 ) +  { +  char *ptr = (char *)p; +  char *end = ptr + n; +  int left = ((long)ptr) & 63; +  +  if( left ) +  { +  low_zero(ptr, left); +  ptr += left; +  } +  +  /* Clear memory without evicting CPU cache. */ +  for( ; ptr <= (end-64); ptr += 64 ) +  { +  __m128i i = _mm_set_epi8(0, 0, 0, 0, +  0, 0, 0, 0, +  0, 0, 0, 0, +  0, 0, 0, 0); +  _mm_stream_si128((__m128i *)&ptr[0], i); +  _mm_stream_si128((__m128i *)&ptr[16], i); +  _mm_stream_si128((__m128i *)&ptr[32], i); +  _mm_stream_si128((__m128i *)&ptr[48], i); +  } +  +  if( end-ptr ) +  low_zero(ptr, end-ptr); +  } +  else + #endif +  low_zero(p, n); + } +    void init_pike_memory (void)   {    init_hashmem(); - #if defined (HAVE_GETPAGESIZE) + #ifdef HAVE_GETPAGESIZE    page_size = getpagesize();   #elif defined (HAVE_SYSCONF) && defined (_SC_PAGESIZE)    page_size = (int) sysconf (_SC_PAGESIZE);   #elif defined (HAVE_SYSCONF) && defined (_SC_PAGE_SIZE)    page_size = (int) sysconf (_SC_PAGE_SIZE);   #elif defined (HAVE_GETSYSTEMINFO)    {    SYSTEM_INFO sysinfo;    GetSystemInfo (&sysinfo);    page_size = sysinfo.dwPageSize;    }   #else    /* A reasonable default... */    page_size = 8192; - #endif + #endif /* HAVE_GETPAGESIZE */      #ifdef INIT_DEV_ZERO    /* Neither MAP_ANONYMOUS nor MAP_ANON.    * Initialize a /dev/zero fd for use with mmap.    */    if (dev_zero < 0) {    if ((dev_zero = open("/dev/zero", O_RDONLY)) < 0) {    fprintf(stderr, "Failed to open /dev/zero.\n");    return NULL;    }