5b75942013-09-21Henrik Grubbström (Grubba) //! A GBM-based storage manager. //! //! This storage manager provides the means to save data to memory. //! In this manager I'll add reference documentation as comments to //! interfaces. It will be organized later in a more comprehensive format //! //! Settings will be added later. //! //! @thanks
0ab1892013-10-05Henrik Grubbström (Grubba) //! Thanks to Francesco Chemolli <kinkie@@roxen.com> for the contribution.
5b75942013-09-21Henrik Grubbström (Grubba) 
2fe7462000-07-02Francesco Chemolli 
a580e12000-09-27Fredrik Hübinette (Hubbe) #pike __REAL_VERSION__
a20af62000-09-26Fredrik Hübinette (Hubbe) 
2fe7462000-07-02Francesco Chemolli //after this many deletion ops, the databases will be compacted.
8b8c2f2001-01-01Francesco Chemolli #define CLUTTERED 1000
2fe7462000-07-02Francesco Chemolli 
633f1d2004-04-17Henrik Grubbström (Grubba) #if constant(Gdbm.gdbm)
2fe7462000-07-02Francesco Chemolli Gdbm.gdbm db, metadb; int deletion_ops=0; //every 1000 deletion ops, we'll reorganize.
8b8c2f2001-01-01Francesco Chemolli int have_dependants=0;
2fe7462000-07-02Francesco Chemolli 
5b75942013-09-21Henrik Grubbström (Grubba) //!
2fe7462000-07-02Francesco Chemolli class Data { inherit Cache.Data; int _size=0; string _key=0; mixed _data=0; int size() {
8b8c2f2001-01-01Francesco Chemolli  return _size; //it's guarranteed to be set. See set()
2fe7462000-07-02Francesco Chemolli  } mixed data() {
8b8c2f2001-01-01Francesco Chemolli  if (!_data)
81c05e2000-07-05Francesco Chemolli  _data=decode_value(db[_key]);
2fe7462000-07-02Francesco Chemolli  return _data; } private inline string metadata_dump () { return encode_value( (["size":_size,"atime":atime, "ctime":ctime,"etime":etime,"cost":cost]) ); } //dumps the metadata if necessary. void sync() {
81c05e2000-07-05Francesco Chemolli  metadb[_key]=metadata_dump();
2fe7462000-07-02Francesco Chemolli  }
81c05e2000-07-05Francesco Chemolli 
2fe7462000-07-02Francesco Chemolli  inline void touch() { atime=time(1); sync(); }
9eaf1d2008-06-28Martin Nilsson  protected void create(string key, Gdbm.gdbm data_db,
cf9d4f2003-11-12Henrik Grubbström (Grubba)  Gdbm.gdbm metadata_db, string dumped_metadata) {
81c05e2000-07-05Francesco Chemolli  mapping m=decode_value(dumped_metadata);
2fe7462000-07-02Francesco Chemolli  _key=key; db=data_db; metadb=metadata_db;
81c05e2000-07-05Francesco Chemolli  _size=m->size; atime=m->atime; ctime=m->ctime; etime=m->etime;
8b8c2f2001-01-01Francesco Chemolli  cost=(float)(m->cost||1);
2fe7462000-07-02Francesco Chemolli  } } //we could maybe use some kind of progressive approach: keep a few //items in queue, then fetch more as we need them. This approach //can be expensive in terms of memory //Something I can't figure out a clean solution for: reorganizing //the database. It would be cool to do that when we know it to be //somewhat junky, but guessing that kind of information would be //quite hard, especially if we consider caches surviving the process //that created them //Maybe we can put some heuristics: since almost only the policy manager //uses first(), next() and delete(), we might count the deletion operations //and reorganize when we reach some kind of threshold.
8b8c2f2001-01-01Francesco Chemolli private string iter=0;
2fe7462000-07-02Francesco Chemolli int(0..0)|string first() {
8b8c2f2001-01-01Francesco Chemolli  return (iter=metadb->firstkey());
2fe7462000-07-02Francesco Chemolli } int(0..0)|string next() {
8b8c2f2001-01-01Francesco Chemolli  return (iter=metadb->nextkey(iter));
2fe7462000-07-02Francesco Chemolli }
81c05e2000-07-05Francesco Chemolli // we set the data in the database directly here, since we have // no need to create a Data object.
2fe7462000-07-02Francesco Chemolli void set(string key, mixed value,
8b8c2f2001-01-01Francesco Chemolli  void|int expire_time, void|float preciousness, void|multiset(string) dependants) {
2fe7462000-07-02Francesco Chemolli  //should I refuse storing objects too? if (programp(value)||functionp(value)) { werror("can't store value\n"); //TODO: use crumbs return 0; }
8b8c2f2001-01-01Francesco Chemolli  string tmp; int tm=time(1); mapping meta;
81c05e2000-07-05Francesco Chemolli  tmp=encode_value(value); db[key]=tmp; meta=(["size":sizeof(tmp),"atime":tm,"ctime":tm]); if (expire_time) meta->etime=expire_time; if (preciousness||!zero_type(preciousness)) meta->cost=preciousness; else meta->cost=1.0;
8b8c2f2001-01-01Francesco Chemolli  if (dependants) { meta->dependants=dependants; have_dependants=1; }
81c05e2000-07-05Francesco Chemolli  metadb[key]=encode_value(meta);
2fe7462000-07-02Francesco Chemolli }
81c05e2000-07-05Francesco Chemolli // we fetch at least the metadata here. If we delegated that to the // Data class, we would waste quite a lot of resources while doing // the undump operation.
2fe7462000-07-02Francesco Chemolli int(0..0)|Cache.Data get(string key,void|int notouch) {
81c05e2000-07-05Francesco Chemolli  string metadata=metadb[key]; Data rv; if (!metadata) return 0; // no such key in cache.
0547e62001-01-02Henrik Grubbström (Grubba)  rv = Data(key,db,metadb,metadata);
81c05e2000-07-05Francesco Chemolli  if (!notouch) { rv->touch();
2fe7462000-07-02Francesco Chemolli  }
81c05e2000-07-05Francesco Chemolli  return rv;
2fe7462000-07-02Francesco Chemolli } //fetches some data from the cache asynchronously. //the callback will get as first argument the key, and as second //argument 0 (cache miss) or a Cache.Data object. void aget(string key, function(string,int(0..0)|Cache.Data:void) callback) { callback(key,get(key)); }
81c05e2000-07-05Francesco Chemolli void delete(string key, void|int(0..1) hard) {
8b8c2f2001-01-01Francesco Chemolli  multiset(string) dependants=0; if (have_dependants) { string emeta=metadb->fetch(key); if (!emeta) return; // no such key. Already deleted. dependants=decode_value(emeta)->dependants; } //werror("Deleteing %s\n",key);
2fe7462000-07-02Francesco Chemolli  db->delete(key); metadb->delete(key); deletion_ops++;
8b8c2f2001-01-01Francesco Chemolli  if (dependants) { foreach((array)dependants, string chain) { //werror("chain-deleteing %s\n",chain); delete(chain); // recursively delete } //werror("Done chain-deleteing\n"); return; // return so that reorg takes place at the end }
2fe7462000-07-02Francesco Chemolli  if (deletion_ops > CLUTTERED) {
8b8c2f2001-01-01Francesco Chemolli  //werror("Reorganizing database\n");
2fe7462000-07-02Francesco Chemolli  db->reorganize(); metadb->reorganize(); deletion_ops=0; } }
5b75942013-09-21Henrik Grubbström (Grubba) //! A GDBM storage-manager must be hooked to a GDBM Database.
2fe7462000-07-02Francesco Chemolli void create(string path) { db=Gdbm.gdbm(path+".db","rwcf"); metadb=Gdbm.gdbm(path+"_meta.db","rwcf"); }
ffaf452004-04-14Martin Nilsson #else constant this_program_does_not_exist=1;
633f1d2004-04-17Henrik Grubbström (Grubba) #endif // constant(Gdbm.gdbm)
2fe7462000-07-02Francesco Chemolli  /**************** thoughts and miscellanea ******************/ //maybe we should split the database into two databases, one for the data //and one for the metadata. //we should really use an in-memory cache for the objects. I delay that //for now, since we don't have a decent footprint-constrained policy //manager yet.