e576bb2002-10-11Martin Nilsson /* || This file is part of Pike. For copyright information see COPYRIGHT. || Pike is distributed under GPL, LGPL and MPL. See the file COPYING || for more information. */
1b10db2002-10-08Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe) /* * Written by Fredrik Hubinette (hubbe@lysator.liu.se) */ #define HSHIFT 0 #include "pike_search_engine2.c" #undef HSHIFT #define HSHIFT 1 #include "pike_search_engine2.c" #undef HSHIFT #define HSHIFT 2 #include "pike_search_engine2.c" #undef HSHIFT #define INTERCASE(NAME,X) \ case X: return MKPCHARP(PxC3(NAME,NSHIFT,X)(s,(PxC(p_wchar,X) *)haystack.ptr,haystacklen),X) #define INTERMEDIATE(NAME) \ PCHARP PxC3(NAME,NSHIFT,N)(void *s, \ PCHARP haystack, \ ptrdiff_t haystacklen) \ { \ switch(haystack.shift) \ { \ INTERCASE(NAME,0); \ INTERCASE(NAME,1); \ INTERCASE(NAME,2); \ } \
5aad932002-08-15Marcus Comstedt  Pike_fatal("Illegal shift\n"); \
9282fd2015-09-27Martin Nilsson  UNREACHABLE(return haystack); \
9b1f032000-10-09Fredrik Hübinette (Hubbe) } \ \
a630c22004-09-18Per Hedbor static const struct SearchMojtVtable PxC3(NAME,NSHIFT,_vtable) = { \
9b1f032000-10-09Fredrik Hübinette (Hubbe)  (SearchMojtFunc0)PxC3(NAME,NSHIFT,0), \ (SearchMojtFunc1)PxC3(NAME,NSHIFT,1), \ (SearchMojtFunc2)PxC3(NAME,NSHIFT,2), \ (SearchMojtFuncN)PxC3(NAME,NSHIFT,N), \ }; INTERMEDIATE(memchr_search) INTERMEDIATE(memchr_memcmp2) INTERMEDIATE(memchr_memcmp3) INTERMEDIATE(memchr_memcmp4) INTERMEDIATE(memchr_memcmp5) INTERMEDIATE(memchr_memcmp6) INTERMEDIATE(boyer_moore_hubbe) INTERMEDIATE(hubbe_search) /* */ int NameN(init_hubbe_search)(struct hubbe_searcher *s, NCHAR *needle, ptrdiff_t needlelen, ptrdiff_t max_haystacklen) { INT32 tmp, h; ptrdiff_t hsize, e, max; NCHAR *q; struct hubbe_search_link *ptr; INT32 linklen[NELEM(s->set)]; INT32 maxlinklength;
865b162000-12-19Henrik Grubbström (Grubba)  s->needle=needle; s->needlelen=needlelen;
9b1f032000-10-09Fredrik Hübinette (Hubbe)  #ifdef PIKE_DEBUG if(needlelen < 7)
5aad932002-08-15Marcus Comstedt  Pike_fatal("hubbe search does not work with search strings shorter than 7 characters!\n");
9b1f032000-10-09Fredrik Hübinette (Hubbe) #endif
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe) #ifdef TUNAFISH hsize=52+(max_haystacklen >> 7) - (needlelen >> 8); max =13+(max_haystacklen >> 4) - (needlelen >> 5);
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  if(hsize > (ptrdiff_t) NELEM(s->set)) { hsize=NELEM(s->set); }else{ for(e=8;e<hsize;e+=e); hsize=e; } #else hsize=NELEM(s->set); max=needlelen; #endif for(e=0;e<hsize;e++) { s->set[e]=0; linklen[e]=0; } hsize--;
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  if(max > (ptrdiff_t)needlelen) max=needlelen;
5a6d8e2006-07-05Martin Stjernholm  max=(max-sizeof(INT32)+1) & ~(sizeof(INT32) - 1);
9b1f032000-10-09Fredrik Hübinette (Hubbe)  if(max > MEMSEARCH_LINKS) max=MEMSEARCH_LINKS;
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  /* This assumes 512 buckets - Hubbe */
b006172014-10-20Martin Nilsson  maxlinklength = (INT32)sqrt((double)max/2)+1;
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  ptr=& s->links[0];
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  q=(NCHAR *)needle;
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe) #if PIKE_BYTEORDER == 4321 && NSHIFT == 0
2838662000-10-14Henrik Grubbström (Grubba)  for(tmp = e = 0; e < (ptrdiff_t)sizeof(INT32)-1; e++)
9b1f032000-10-09Fredrik Hübinette (Hubbe)  { tmp<<=8; tmp|=*(q++); } #endif
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  for(e=0;e<max;e++) { #if PIKE_BYTEORDER == 4321 && NSHIFT == 0 tmp<<=8; tmp|=*(q++); #else /* FIXME tmp=EXTRACT_INT(q); */ tmp=NameN(GET_4_UNALIGNED_CHARS)(q); q++; #endif h=tmp; h+=h>>7; h+=h>>17; h&=hsize;
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  ptr->offset=e; ptr->key=tmp; ptr->next=s->set[h]; s->set[h]=ptr; ptr++; linklen[h]++;
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  if(linklen[h] > maxlinklength) { return 0; }
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  } s->hsize=hsize; s->max=max; return 1; }
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  void NameN(init_boyer_moore_hubbe)(struct boyer_moore_hubbe_searcher *s, NCHAR *needle, ptrdiff_t needlelen, ptrdiff_t max_haystacklen) { ptrdiff_t e;
865b162000-12-19Henrik Grubbström (Grubba)  s->needle=needle; s->needlelen=needlelen;
9b1f032000-10-09Fredrik Hübinette (Hubbe)  #ifdef PIKE_DEBUG if(needlelen < 2)
8313ee2003-04-23Johan Sundström  Pike_fatal("boyer-moore-hubbe search does not work with single-character search strings!\n");
9b1f032000-10-09Fredrik Hübinette (Hubbe) #endif
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe) #ifdef TUNAFISH s->plen = 8 + ((max_haystacklen-needlelen) >> 5); if(s->plen>needlelen) s->plen=needlelen; if(s->plen>BMLEN) s->plen=BMLEN; #else s->plen=BMLEN; if(s->plen>needlelen) s->plen=needlelen; #endif
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  for(e=0;e<(ptrdiff_t)NELEM(s->d1);e++) s->d1[e]=s->plen;
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  for(e=0;e<s->plen;e++) { s->d1[ PxC3(BMHASH,NSHIFT,0) (needle[e]) ]=s->plen-e-1; s->d2[e]=s->plen*2-e-1; }
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  for(e=0;e<s->plen;e++) { ptrdiff_t d; for(d=0;d<=e && needle[s->plen-1-d]==needle[e-d];d++);
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  if(d>e) { while(s->plen-1-d>=0) { s->d2[s->plen-1-d]=s->plen-e+d-1; d++; } }else{ s->d2[s->plen-1-d]=s->plen-e+d-1; } } } void NameN(init_memsearch)( struct pike_mem_searcher *s, NCHAR *needle, ptrdiff_t needlelen, ptrdiff_t max_haystacklen) { switch(needlelen) { case 0: s->mojt.vtab=&nil_search_vtable; return; case 1:
ae0ca82000-10-10Henrik Grubbström (Grubba)  s->mojt.data=(void *)(ptrdiff_t)(needle[0]);
9b1f032000-10-09Fredrik Hübinette (Hubbe)  s->mojt.vtab=& PxC3(memchr_search,NSHIFT,_vtable); return; #define MMCASE(X) \ case X: \ s->mojt.data=(void *) needle; \ s->mojt.vtab=& PxC4(memchr_memcmp,X,NSHIFT,_vtable); \ return MMCASE(2); MMCASE(3); MMCASE(4); MMCASE(5); MMCASE(6);
e3b79d2001-01-19Henrik Grubbström (Grubba) #undef MMCASE
9b1f032000-10-09Fredrik Hübinette (Hubbe)  case 7: case 8: case 9: case 10: case 11: case 12: case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20: case 21: case 22: case 23: case 24: case 25: case 26: case 27: case 28: case 29: case 30: case 31: case 32: case 33: case 34: break; default: if(max_haystacklen > needlelen + 64) { if(NameN(init_hubbe_search)(& s->data.hubbe, needle,needlelen, max_haystacklen)) { s->mojt.vtab=& PxC3(hubbe_search,NSHIFT,_vtable); s->mojt.data=(void *)& s->data.hubbe; return; } } } NameN(init_boyer_moore_hubbe)(& s->data.bm, needle,needlelen, max_haystacklen); s->mojt.vtab=& PxC3(boyer_moore_hubbe,NSHIFT,_vtable); s->mojt.data=(void *)& s->data.bm; } SearchMojt NameN(compile_memsearcher)(NCHAR *needle,
ae0ca82000-10-10Henrik Grubbström (Grubba)  ptrdiff_t len,
9b1f032000-10-09Fredrik Hübinette (Hubbe)  int max_haystacklen, struct pike_string *hashkey) { /* FIXME: * if NSHIFT > 0 but all characters in needle can fit in * a narrower string, it might pay off to reduce the width * of the string. */ if(len<7) { struct pike_mem_searcher tmp; NameN(init_memsearch)(&tmp, needle,len, max_haystacklen);
b6b7ee2008-06-23Martin Stjernholm  tmp.mojt.container = NULL;
9b1f032000-10-09Fredrik Hübinette (Hubbe)  return tmp.mojt; }else{ struct svalue *sval,stmp; struct pike_mem_searcher *s; struct object *o;
13670c2015-05-25Martin Nilsson 
9b1f032000-10-09Fredrik Hübinette (Hubbe)  if(!hashkey) hashkey=NameN(make_shared_binary_string)(needle,len); else add_ref(hashkey); if((sval=low_mapping_string_lookup(memsearch_cache,hashkey))) {
017b572011-10-28Henrik Grubbström (Grubba)  if(TYPEOF(*sval) == T_OBJECT)
9b1f032000-10-09Fredrik Hübinette (Hubbe)  { o=sval->u.object; if(o->prog == pike_search_program) { s=OB2MSEARCH(sval->u.object);
b6b7ee2008-06-23Martin Stjernholm  assert (sval->u.object == s->mojt.container);
9b1f032000-10-09Fredrik Hübinette (Hubbe)  add_ref(sval->u.object); free_string(hashkey); return s->mojt; } }
a9af932012-09-26Henrik Grubbström (Grubba)  /* Paranoia: Junk entry in the mapping. Remove it. */ SET_SVAL(stmp, T_STRING, 0, string, hashkey); map_delete(memsearch_cache, &stmp); } if (memsearch_cache->data->size >= memsearch_cache_threshold) { /* Perform a gc of all Search objects that only are * referenced from the cache. * Allow the cache to grow to double the size before * the next synchronous gc. */ struct keypair *k = NULL; struct mapping_data *md = memsearch_cache->data; int e; int count = 0; /* NB: We inline some stuff from mapping.c here to avoid copying the md. */ for (e=0; e < md->hashsize; e++) { struct keypair **prev; for(prev = md->hash + e; (k = *prev);) { count++;
4a93e82013-06-11Henrik Grubbström (Grubba)  if (REFCOUNTED_TYPE(TYPEOF(k->val)) &&
a9af932012-09-26Henrik Grubbström (Grubba)  (*k->val.u.refs == 1)) { /* map_delete(memsearch_cache, &k->ind); */ *prev = k->next; free_svalue(&k->ind); free_svalue(&k->val); mapping_free_keypair(md, k); md->size--; continue; } else if (count < 10) { // locate_references(k->val.u.refs); } prev = &k->next; } } memsearch_cache_threshold = (memsearch_cache->data->size<<1) | 1; if (memsearch_cache_threshold < MIN_MEMSEARCH_THRESHOLD) { memsearch_cache_threshold = MIN_MEMSEARCH_THRESHOLD; } } else if ((memsearch_cache_threshold & 1) && (memsearch_cache->data->size<<2 < memsearch_cache_threshold)) { /* The real gc() has run and zapped some of our entries. * Assume that all entries left in the cache have more than * one reference. */ memsearch_cache_threshold = (memsearch_cache->data->size<<1) | 1; if (memsearch_cache_threshold < MIN_MEMSEARCH_THRESHOLD) { memsearch_cache_threshold = MIN_MEMSEARCH_THRESHOLD; }
9b1f032000-10-09Fredrik Hübinette (Hubbe)  } o=low_clone(pike_search_program); s=OB2MSEARCH(o);
b6b7ee2008-06-23Martin Stjernholm  s->mojt.container = o; /* Not refcounted self-ref. */
9b1f032000-10-09Fredrik Hübinette (Hubbe)  s->s=hashkey; /* We use 0x7fffffff for max_haystacklen because we do * not know how many times this search struct will be * reused. */ NameN(init_memsearch)(s, needle,len, 0x7fffffff);
017b572011-10-28Henrik Grubbström (Grubba)  SET_SVAL(stmp, T_OBJECT, 0, object, o);
9b1f032000-10-09Fredrik Hübinette (Hubbe)  mapping_string_insert(memsearch_cache, hashkey, &stmp); return s->mojt; } }