Roxen.git / server / base_server / fsgc.pike

version» Context lines:

Roxen.git/server/base_server/fsgc.pike:1: + // + // Unified file system garbage collector. + // + // 2013-09-12 Henrik Grubbström + //    -  + #if constant(Filesystem.Monitor.basic) +  + // #define FSGC_DEBUG + // #define FSGC_PRETEND +  + #ifdef FSGC_DEBUG + #define GC_WERR(X...) werror(X) + #else + #define GC_WERR(X...) + #endif +  + /* Some notes: +  * +  * There are multiple data for a file that may affect the garbage policy: +  * +  * * The age of the file. +  * +  * * The size of the file. +  * +  * Garbage collection for a root may be triggered by several factors: +  * +  * * A maxium age for a file has been reached. +  * +  * * Too many files under a root. +  * +  * * The total size of the files under a root is too large. +  * +  * Symlinks are not followed and not garbage collected due +  * to the inherent risks of escaping directory structures +  * and/or removing manually added stuff. +  */ +  + //! Filesystem garbage collector for a single root directory. + class FSGarb + { +  inherit Filesystem.Monitor.basic : basic; +  +  int num_files; +  int total_size; +  +  string modid; +  string root; +  int max_age; +  int max_files; +  int max_size; +  +  mapping(string:object) handle_lookup = ([]); +  ADT.Priority_queue pending_gc = ADT.Priority_queue(); +  +  //! If set, move files to this directory instead of deleting them. +  //! +  //! If set to @[root] or @expr{""@} keep the files as is. +  string quarantine; +  +  protected int rm(string path) +  { +  GC_WERR("FSGC: Zap %O\n", path); +  if (quarantine) { +  if ((quarantine == root) || (quarantine == "")) return 0; +  if (!has_prefix(path, root)) return 0; +  string rel = path[sizeof(root)..]; +  +  // First try the trivial case. +  if (mv(path, quarantine + rel)) return 1; +  +  string dirs = dirname(rel); +  if (sizeof(dirs)) { +  if (Stdio.mkdirhier(quarantine + dirs)) { +  // Try again with the directory existing. +  if (mv(path, quarantine + rel)) return 1; +  } +  } +  +  // Different filesystems? +  if (Stdio.cp(path, quarantine + rel)) { +  return predef::rm(path); +  } +  werror("FSGC: Failed to copy file %O to %O: %s.\n", +  path, quarantine + rel, strerror(errno())); +  return 0; +  } else { +  return predef::rm(path); +  } +  } +  +  void check_threshold() +  { +  GC_WERR("FSGC: Checking thresholds...\n" +  " total_size: %d max_size: %d\n" +  " num_files: %d max_files: %d\n", +  total_size, max_size, +  num_files, max_files); +  +  while ((max_size && (total_size > max_size)) || +  (max_files && (num_files > max_files))) { +  GC_WERR("FSGC: Filesystem limits exceeded forcing early removal.\n"); +  if (!zap_one_file()) break; +  } +  } +  +  protected int zap_one_file() +  { +  if (!sizeof(pending_gc)) return 0; +  +  // Pop the next pending file from the queue. +  Monitor m = pending_gc->pop(); +  m_delete(handle_lookup, m->path); +  +  // Account for the deletion immediately, and +  // make sure it isn't counted twice. +  int bytes = m->st->size; +  m->st->size = 0; +  m->st->isreg = 0; +  +  GC_WERR("Deleting file %O...\n", m->path); +  if (rm(m->path)) { +  num_files--; +  total_size -= bytes; +  +  // Make sure the deletion is notified properly soon. +  m->next_poll = time(1); +  monitor_queue->adjust(m); +  } else { +  GC_WERR("Failed to delete file %O: %s\n", +  m->path, strerror(errno())); +  // Restore the state in case the file is altered externally. +  m->st->size = bytes; +  m->st->isreg = 1; +  } +  return 1; +  } +  +  int st_to_pri(Stdio.Stat st) +  { +  return st->mtime - st->size / 1024; +  } +  +  protected void remove_pending(Monitor m) +  { +  // Register us for threshold-based deletion. +  object handle = m_delete(handle_lookup, m->path); +  if (handle) { +  pending_gc->adjust_pri(handle, -0x80000000); +  pending_gc->pop(); +  } +  } +  +  protected class Monitor { +  inherit basic::Monitor; +  +  protected void create(string path, +  MonitorFlags flags, +  int max_dir_check_interval, +  int file_interval_factor, +  int stable_time) +  { +  ::create(path, flags, max_dir_check_interval, +  file_interval_factor, stable_time); +  GC_WERR("%O->create(%O, %O, %O, %O, %O)\n", +  this_object(), path, flags, max_dir_check_interval, +  file_interval_factor, stable_time); +  } +  +  void check_for_release(int mask, int flags) +  { +  GC_WERR("%O->check_for_relase(0x%x, 0x%x)\n", +  this_object(), mask, flags); +  ::check_for_release(mask, flags); +  if (!monitors[path]) { +  // We've been relased. +  // Make sure to update our parent (if any) soon. +  array a = path/"/"; +  Monitor m = monitors[canonic_path(a[..sizeof(a)-2]*"/")]; +  if (m) { +  GC_WERR("Waking up our parent dir: %O\n", m); +  m->next_poll = time(1)-1; +  monitor_queue->adjust(m); +  } +  } +  } +  +  protected void file_exists(string path, Stdio.Stat st) +  { +  ::file_exists(path, st); +  // Make sure we get the stable change callback... +  last_change = st->mtime; +  +  if (st->isreg) { +  num_files++; +  total_size += st->size; +  +  // Register us for threadhold-based deletion. +  handle_lookup[path] = pending_gc->push(st_to_pri(st), this); +  +  check_threshold(); +  } +  } +  +  // NB: Needs to be visible so that reconfigure() can call it. +  void update(Stdio.Stat st) +  { +  int delta = max_dir_check_interval || basic::max_dir_check_interval; +  if (!next_poll) { +  // Attempt to distribute polls evenly at startup. +  delta = 1 + random(delta); +  if (st) { +  last_change = st->mtime; +  } +  } +  +  ::update(st); +  +  // We're only interested in stable time, so there's no reason +  // to scan as frequently as the default implementation. +  +  if (last_change <= time(1)) { +  // Time until stable. +  int d = last_change + (stable_time || basic::stable_time) - time(1); +  +  GC_WERR("%O: last: %s, d: %d, delta: %d\n", +  this_object(), ctime(last_change) - "\n", d, delta); +  if (d < 0) d = 1; +  if (d < delta) delta = d; +  } +  next_poll = time(1) + (delta || 1); +  GC_WERR("%O->update(%O) ==> next: %s\n", +  this_object(), st, ctime(next_poll) - "\n"); +  monitor_queue->adjust(this); +  } +  +  protected string _sprintf(int c) +  { +  return sprintf("FSGarb.Monitor(%O, %O, last: %d, next: %s, st: %O)", +  path, flags, last_change, ctime(next_poll) - "\n", st); +  } +  +  int(0..1) check(MonitorFlags|void flags) +  { +  int(0..1) ret = ::check(flags); +  return ret; +  } +  +  int(0..1) status_change(Stdio.Stat old_st, Stdio.Stat st, +  MonitorFlags old_flags, MonitorFlags flags) +  { +  GC_WERR("Status change %O(0x%x) ==> %O(0x%x) for %O!\n", +  old_st, old_flags, st, flags, this_object()); +  int res = ::status_change(old_st, st, old_flags, flags); +  if (st->isdir && (flags & MF_RECURSE)) { +  foreach(files, string file) { +  file = canonic_path(Stdio.append_path(path, file)); +  if (!monitors[file]) { +  // Lost update due to race-condition: +  // +  // Exist ==> Deleted ==> Exists +  // +  // with no update of directory inbetween. +  // +  // Create the lost submonitor again. +  res = 1; +  monitor(file, old_flags | MF_AUTO | MF_HARD, +  max_dir_check_interval, +  file_interval_factor, +  stable_time); +  monitors[file]->check(); +  } +  } +  } +  +  num_files += st->isreg - old_st->isreg; +  +  if (old_st->isreg) { +  total_size -= old_st->size; +  +  if (!st->isreg) { +  remove_pending(this); +  } +  } +  if (st->isreg) { +  total_size += st->size; +  +  // Register us for threshold-based deletion. +  if (!old_st->isreg) { +  handle_lookup[path] = pending_gc->push(st_to_pri(st), this); +  } else { +  object handle = handle_lookup[path]; +  if (handle && (st_to_pri(st) != st_to_pri(old_st))) { +  pending_gc->adjust_pri(handle, st_to_pri(st)); +  } +  } +  } +  +  check_threshold(); +  return res; +  } +  +  void file_created(string path, Stdio.Stat st) +  { +  GC_WERR("File %O %O created (%O).\n", path, st, this_object()); +  +  if (st->isreg) { +  num_files++; +  total_size += st->size; +  +  // Register us for threshold-based deletion. +  handle_lookup[path] = pending_gc->push(st_to_pri(st), this); +  +  check_threshold(); +  } +  } +  +  void file_deleted(string path, Stdio.Stat old_st) +  { +  GC_WERR("File %O %O deleted (%O).\n", path, old_st, this_object()); +  +  if (old_st->isreg) { +  num_files--; +  total_size -= old_st->size; +  +  remove_pending(this); +  +  check_threshold(); +  } +  } +  } +  +  constant DefaultMonitor = Monitor; +  +  protected void create(string modid, string path, int max_age, +  int|void max_size, int|void max_files, +  string|void quarantine) +  { +  GC_WERR("FSGC: Max age: %d\n", max_age); +  GC_WERR("FSGC: Max size: %d\n", max_size); +  GC_WERR("FSGC: Max files: %d\n", max_files); +  +  this_program::modid = modid; +  +  this_program::max_age = max_age; +  this_program::max_size = max_size; +  this_program::max_files = max_files; +  +  root = canonic_path(path); +  +  if (quarantine) { +  if (sizeof(quarantine)) { +  quarantine = canonic_path(quarantine); +  } +  this::quarantine = quarantine; +  } +  +  ::create(max_age/file_interval_factor, 0, max_age); +  +  // Workaround for too strict type-check in Pike 7.8. +  int flags = 3; +  +  monitor(root, flags); +  } +  +  void stable_data_change(string path, Stdio.Stat st) +  { +  if (path == root) return; +  GC_WERR("FSGC: Deleting stale file: %O\n", path); +  + #if 0 +  // If we ever use accelerated notifications again. +  +  // Override accelerated stable change notification. +  if (st->mtime >= time(1) - stable_time) { +  GC_WERR("FSGC: Keeping file: %O\n", path); +  // Remove the stable notification marker, and reschedule. +  Monitor m = monitor(path, MF_AUTO); +  m->last_change = st->mtime; +  // m->update(st); +  m->check(); // Force an update(). +  return; +  } + #endif +  rm(path); +  } +  +  void reconfigure(int new_max_age, int|void new_max_size, +  int|void new_max_files) +  { +  if (!zero_type(new_max_size)) { +  GC_WERR("FSGC: New max size: %d\n", new_max_size); +  max_size = new_max_size; +  } +  if (!zero_type(new_max_files)) { +  GC_WERR("FSGC: New max files: %d\n", new_max_files); +  max_files = new_max_files; +  } +  if (new_max_age != max_age) { +  GC_WERR("FSGC: New max age: %d\n", new_max_age); +  this_program::max_age = new_max_age; +  int old_stable_time = stable_time; +  set_max_dir_check_interval(stable_time = new_max_age); +  if (stable_time < old_stable_time) { +  // We need to adjust the scan times for the monitors. +  foreach(values(monitors), Monitor m) { +  m->next_poll = 0; +  m->update(m->st); +  } +  } +  } +  +  check_threshold(); +  } +  +  int check(mixed ... args) +  { +  int res = ::check(@args); +  GC_WERR("FSGC: check(%{%O, %}) ==> %O\n", args, res); +  return res; +  } +  +  protected string _sprintf(int c, mapping|void opts) +  { +  return sprintf("FSGarb(%O, %d)", root, stable_time); +  } +  +  array(Stdio.Stat) get_stats() +  { +  return filter(values(monitors)->st, +  lambda(Stdio.Stat st) { +  return st && st->isreg; +  }); +  } + } +  + mapping(string:FSGarb) fsgarbs = ([]); +  + Thread.Thread meta_fsgc_thread; +  + void meta_fsgc() + { +  // Sleep a bit to avoid the startup race. +  sleep(60); +  while(meta_fsgc_thread) { +  int max_sleep = 60; +  foreach(fsgarbs; string id; FSGarb g) { +  int seconds = g && g->check(); +  if (seconds < max_sleep) max_sleep = seconds; +  } +  if (max_sleep < 1) max_sleep = 1; +  GC_WERR("FSGC: Sleeping %d seconds...\n", max_sleep); +  while(meta_fsgc_thread && max_sleep--) { +  sleep(1); +  } +  } + } +  + //! Wrapper keeping a @[FSGarb] alive. + //! + //! When this object is destructed (eg by refcount), the corresponding + //! @[FSGarb] will be killed. This is to make sure stale @[FSGarb]s aren't + //! left running after module reloads or reconfigurations. + class FSGarbWrapper(string id) + { +  protected void destroy() +  { +  GC_WERR("FSGC: FSGarbWrapper %O destructed.\n", id); +  FSGarb g = m_delete(fsgarbs, id); +  if (g) destruct(g); +  } +  +  protected string _sprintf(int c, mapping|void opts) +  { +  return sprintf("FSGarbWrapper(%O)", id); +  } +  +  void reconfigure(int max_age, int|void max_size, int|void max_files) +  { +  FSGarb g = fsgarbs[id]; +  if (g) g->reconfigure(max_age, max_size, max_files); +  } + } +  + FSGarbWrapper register_fsgarb(string modid, string path, int max_age, +  int|void max_size, int|void max_files, +  string|void quarantine) + { +  if ((path == "") || (path == "/") || (max_age <= 0)) return 0; +  string id = modid + "\0" + path + "\0" + gethrtime(); +  FSGarb g = FSGarb(modid, path, max_age, max_size, max_files, +  quarantine); +  fsgarbs[id] = g; +  GC_WERR("FSGC: Register garb on %O ==> id: %O\n", path, id); +  return FSGarbWrapper(id); + } +  + void name_thread(object thread, string name); +  + protected void start_fsgarb() + { +  meta_fsgc_thread = Thread.Thread(meta_fsgc); +  name_thread(meta_fsgc_thread, "Filesystem GC"); + } +  + protected void stop_fsgarb() + { +  Thread.Thread th = meta_fsgc_thread; +  if (th) { +  meta_fsgc_thread = UNDEFINED; +  th->wait(); +  name_thread(th, UNDEFINED); +  } + } +  + #endif /* Filesystem.Monitor.basic */   Newline at end of file added.