5b75942013-09-21Henrik Grubbström (Grubba) //! A Yabu-based storage manager. //! //! Settings will be added later. //! //! @thanks
0ab1892013-10-05Henrik Grubbström (Grubba) //! Thanks to Francesco Chemolli <kinkie@@roxen.com> for the contribution.
2fe7462000-07-02Francesco Chemolli 
a580e12000-09-27Fredrik Hübinette (Hubbe) #pike __REAL_VERSION__
a20af62000-09-26Fredrik Hübinette (Hubbe) 
2fe7462000-07-02Francesco Chemolli #define CLUTTERED 200
3ddaac2014-08-25Per Hedbor inherit Cache.Storage.Base;
2fe7462000-07-02Francesco Chemolli Yabu.Table db, metadb;
8fa8b42014-08-11Martin Nilsson Yabu.DB yabudb;
2fe7462000-07-02Francesco Chemolli  int deletion_ops=0;
8b8c2f2001-01-01Francesco Chemolli int have_dependants=0;
2fe7462000-07-02Francesco Chemolli 
5b75942013-09-21Henrik Grubbström (Grubba) //!
2fe7462000-07-02Francesco Chemolli class Data { inherit Cache.Data; //metadata is kept around, data loaded on demand. int _size=0; string _key=0; mixed _data=0;
36eafd2000-11-26Martin Nilsson  private Yabu.Table db, metadb;
3524712015-05-26Martin Nilsson 
2fe7462000-07-02Francesco Chemolli  int size() {
8b8c2f2001-01-01Francesco Chemolli  return _size; // it's guarranteed to be computed in set()
2fe7462000-07-02Francesco Chemolli  }
3524712015-05-26Martin Nilsson 
2fe7462000-07-02Francesco Chemolli  mixed data() {
3524712015-05-26Martin Nilsson  if (!_data)
2fe7462000-07-02Francesco Chemolli  _data=db->get(_key); return _data; }
3524712015-05-26Martin Nilsson 
2fe7462000-07-02Francesco Chemolli  private inline mapping metadata_dump () { return (["size":_size,"atime":atime, "ctime":ctime,"etime":etime,"cost":cost]); }
3524712015-05-26Martin Nilsson 
2fe7462000-07-02Francesco Chemolli  //dumps the metadata if necessary. void sync() { metadb->set(_key,metadata_dump()); } inline void touch() { atime=time(1); sync(); }
8b8c2f2001-01-01Francesco Chemolli  //m contains the metadata
3524712015-05-26Martin Nilsson  void create(string key, Yabu.Table data_db, Yabu.Table metadata_db,
8b8c2f2001-01-01Francesco Chemolli  mapping m) {
2fe7462000-07-02Francesco Chemolli  _key=key;
eea3102000-12-14Martin Nilsson  db=data_db; metadb=metadata_db;
8b8c2f2001-01-01Francesco Chemolli  _size=m->size; atime=m->atime; ctime=m->ctime; etime=m->etime; cost=m->cost||1.0;
2fe7462000-07-02Francesco Chemolli  }
3524712015-05-26Martin Nilsson 
2fe7462000-07-02Francesco Chemolli } //we could maybe use some kind of progressive approach: keep a few //items in queue, then fetch more as we need them. This approach //can be expensive in terms of memory //Something I can't figure out a clean solution for: reorganizing //the database. It would be cool to do that when we know it to be //somewhat junky, but guessing that kind of information would be //quite hard, especially if we consider caches surviving the process //that created them //Maybe we can put some heuristics: since almost only the policy manager //uses first(), next() and delete(), we might count the deletion operations //and reorganize when we reach some kind of threshold.
8b8c2f2001-01-01Francesco Chemolli private multiset(string) keys;
2fe7462000-07-02Francesco Chemolli int(0..0)|string first() {
8b8c2f2001-01-01Francesco Chemolli  keys=mkmultiset(indices(metadb)); string rv=indices(keys)[0]; keys[rv]=0; return rv;
2fe7462000-07-02Francesco Chemolli } int(0..0)|string next() {
8b8c2f2001-01-01Francesco Chemolli  if (!keys || !sizeof(keys)) return 0; string rv=indices(keys)[0]; keys[rv]=0; return rv;
2fe7462000-07-02Francesco Chemolli } void set(string key, mixed value,
8b8c2f2001-01-01Francesco Chemolli  void|int expire_time, void|float preciousness, void|multiset(string) dependants) {
2fe7462000-07-02Francesco Chemolli  //problem: we can't store objects, functions or programs. //we check here for the actual type. BUT if some 'forbidden' type //is in a composite type's element, we'll mangle it along the way. //Checking for bad types in composite arguments would be very expensive //so I'd favour just stating the problem in the documentation and let //developers take care of this themselves. if (programp(value)||functionp(value)||objectp(value)) { werror("can't store value\n"); //TODO: use crumbs return 0; }
817ad02000-07-05Francesco Chemolli  int tm=time(1); mapping meta; db->set(key,value); //maybe we could lazy-ify this meta=(["size":sizeof(encode_value(value)), "atime":tm,"ctime":tm]); if (expire_time) meta->etime=expire_time;
65340d2014-08-15Martin Nilsson  if (preciousness||!undefinedp(preciousness))
817ad02000-07-05Francesco Chemolli  meta->cost=preciousness; else meta->cost=1.0;
8b8c2f2001-01-01Francesco Chemolli  if (dependants) { meta->dependants=dependants; have_dependants=1; }
817ad02000-07-05Francesco Chemolli  metadb->set(key,meta);
2fe7462000-07-02Francesco Chemolli } int(0..0)|Cache.Data get(string key,void|int notouch) { mixed tmp=metadb->get(key);
8b8c2f2001-01-01Francesco Chemolli  if (!tmp) return 0; tmp=Data(key, db, metadb, tmp); if (!notouch) { tmp->touch();
2fe7462000-07-02Francesco Chemolli  } return tmp; } //fetches some data from the cache asynchronously. //the callback will get as first argument the key, and as second //argument 0 (cache miss) or a Cache.Data object. void aget(string key, function(string,int(0..0)|Cache.Data:void) callback) { callback(key,get(key)); }
817ad02000-07-05Francesco Chemolli void delete(string key, void|int(0..1) hard) {
8b8c2f2001-01-01Francesco Chemolli  multiset dependants=0;
3524712015-05-26Martin Nilsson 
8b8c2f2001-01-01Francesco Chemolli  if (have_dependants) { mapping emeta=metadb->get(key); if (!emeta) return; dependants=emeta->dependants; }
3524712015-05-26Martin Nilsson 
8b8c2f2001-01-01Francesco Chemolli  if (keys) keys[key]=0; db->delete(key); // maybe we should be transactional?
2fe7462000-07-02Francesco Chemolli  metadb->delete(key); deletion_ops++;
3524712015-05-26Martin Nilsson 
8b8c2f2001-01-01Francesco Chemolli  if (have_dependants && dependants && sizeof(dependants)) { foreach((array)dependants, string chain) { //werror("chain-deleteing %s\n",chain); delete(chain); // recursively delete. } //werror("done chain-deleting\n"); return; }
3524712015-05-26Martin Nilsson 
2fe7462000-07-02Francesco Chemolli  if (deletion_ops > CLUTTERED) { yabudb->reorganize(); deletion_ops=0; } }
5b75942013-09-21Henrik Grubbström (Grubba) //!
2fe7462000-07-02Francesco Chemolli void create(string path) {
8fa8b42014-08-11Martin Nilsson  yabudb=Yabu.DB(path+".yabu","wcSQ"); //let's hope I got the mode right.
2fe7462000-07-02Francesco Chemolli  db=yabudb["data"]; metadb=yabudb["metadata"]; } /**************** thoughts and miscellanea ******************/ //maybe we should split the database into two databases, one for the data //and one for the metadata. //we should really use an in-memory cache for the objects. I delay that //for now, since we don't have a decent footprint-constrained policy //manager yet.