Merge pull request #56 from larsewi/purge

CFE-4373: Implemented new purge functionality

bin/Makefile.am

@@ -9,6 +9,7 @@ leech_SOURCES = main.c \
     diff.c diff.h \
     rebase.c rebase.h \
     patch.c patch.h \
-    history.c history.h
+    history.c history.h \
+    purge.c purge.h
 leech_LDADD = @PSQL_LIBS@ $(top_builddir)/lib/libleech.la
 leech_CFLAGS = @PSQL_CFLAGS@

bin/main.c

@@ -12,6 +12,7 @@
 #include "diff.h"
 #include "history.h"
 #include "patch.h"
+#include "purge.h"
 #include "rebase.h"
 
 #ifdef HAVE_LIBPQ
@@ -54,6 +55,7 @@ static const struct command COMMANDS[] = {
     {"rebase", "rebase to current table state", Rebase},
     {"patch", "apply changes to tables", Patch},
     {"history", "get history of a specific record", History},
+    {"purge", "delete old/unreachable blocks", Purge},
     {NULL, NULL, NULL},
 };
 

bin/purge.c

@@ -0,0 +1,52 @@
+#include "purge.h"
+
+#include <stdio.h>
+
+#include "common.h"
+
+enum OPTION_VALUE {
+  OPTION_HELP = 1,
+};
+
+struct arguments {
+  const char *arg;
+  const char *desc;
+};
+
+static const struct option OPTIONS[] = {
+    {"help", no_argument, NULL, OPTION_HELP},
+    {NULL, 0, NULL, 0},
+};
+
+static const char *const DESCRIPTIONS[] = {
+    "print help message",
+};
+
+static void PrintHelp(void) {
+  PrintVersion();
+  printf("\n");
+  PrintOptions(OPTIONS, DESCRIPTIONS);
+  printf("\n");
+  PrintBugreport();
+  printf("\n");
+}
+
+int Purge(const char *const work_dir, int argc, char *argv[]) {
+  int opt;
+  while ((opt = getopt_long(argc, argv, "+", OPTIONS, NULL)) != -1) {
+    switch (opt) {
+      case OPTION_HELP:
+        PrintHelp();
+        return EXIT_SUCCESS;
+      default:
+        return EXIT_FAILURE;
+    }
+  }
+
+  if (!LCH_Purge(work_dir)) {
+    fprintf(stderr, "Failed to purge blocks");
+    return EXIT_FAILURE;
+  }
+
+  return EXIT_SUCCESS;
+}

bin/purge.h

@@ -0,0 +1,6 @@
+#ifndef _LEECH_PURGE_H
+#define _LEECH_PURGE_H
+
+int Purge(const char *work_dir, int argc, char *argv[]);
+
+#endif  // _LEECH_PURGE_H

configure.ac

@@ -46,7 +46,7 @@ AM_CONDITIONAL([BUILD_PSQL_MODULE], [test "x$with_psql_module" = "xyes"])
 AM_CONDITIONAL([BUILD_CSV_MODULE], [test "x$with_csv_module" = "xyes"])
 
 # Config constants.
-AC_DEFINE([LCH_DEFAULT_MAX_CHAIN_LENGTH], 2048, [Default max chain length used in block garbage collector])
+AC_DEFINE([LCH_DEFAULT_PREFERED_CHAIN_LENGTH], 2048, [Default prefered chain length used in block garbage collector])
 AC_DEFINE([LCH_JSON_PRETTY_INDENT_SIZE], 2, [Indent size used when composing pretty JSON])
 AC_DEFINE([LCH_BUFFER_SIZE], 1024, [Initial buffer size allocated by leech])
 AC_DEFINE([LCH_DICT_CAPACITY], 256, [Initial dictionary capacity allocated by leech])

lib/block.c

@@ -68,7 +68,7 @@ bool LCH_BlockStore(const LCH_Instance *const instance,
   assert(block != NULL);
 
   const char *const work_dir = LCH_InstanceGetWorkDirectory(instance);
-  const bool pretty_print = LCH_InstancePrettyPrint(instance);
+  const bool pretty_print = LCH_InstanceShouldPrettyPrint(instance);
 
   LCH_Buffer *const json = LCH_JsonCompose(block, pretty_print);
   if (json == NULL) {

lib/files.c

@@ -1,6 +1,7 @@
 #include "files.h"
 
 #include <assert.h>
+#include <dirent.h>
 #include <errno.h>
 #include <libgen.h>
 #include <stdarg.h>
@@ -9,7 +10,6 @@
 #include <unistd.h>
 
 #include "definitions.h"
-#include "list.h"
 #include "logger.h"
 #include "string_lib.h"
 
@@ -152,3 +152,49 @@ bool LCH_FileCreateParentDirectories(const char *const filename) {
   LCH_ListDestroy(dirs);
   return true;
 }
+
+LCH_List *LCH_FileListDirectory(const char *const path,
+                                const bool filter_hidden) {
+  LCH_List *const filenames = LCH_ListCreate();
+  if (filenames == NULL) {
+    return NULL;
+  }
+
+  DIR *dir = opendir(path);
+  if (dir == NULL) {
+    LCH_LOG_ERROR("Failed to open directory '%s': %s", path, strerror(errno));
+    LCH_ListDestroy(filenames);
+    return NULL;
+  }
+
+  errno = 0;  // Only way to distinguish between error or end-of-directory
+  struct dirent *entry = NULL;
+  while ((entry = readdir(dir)) != NULL) {
+    if (filter_hidden && LCH_StringStartsWith(entry->d_name, ".")) {
+      continue;
+    }
+
+    char *const filename = LCH_StringDuplicate(entry->d_name);
+    if (filename == NULL) {
+      LCH_ListDestroy(filenames);
+      closedir(dir);
+      return NULL;
+    }
+
+    if (!LCH_ListAppend(filenames, filename, free)) {
+      LCH_ListDestroy(filenames);
+      closedir(dir);
+      return NULL;
+    }
+  }
+
+  if (errno != 0) {
+    LCH_LOG_ERROR("Failed to read directory '%s': %s", path, strerror(errno));
+    LCH_ListDestroy(filenames);
+    closedir(dir);
+    return NULL;
+  }
+
+  closedir(dir);
+  return filenames;
+}

lib/files.h

@@ -4,6 +4,8 @@
 #include <stdbool.h>
 #include <stdio.h>
 
+#include "list.h"
+
 bool LCH_FileSize(FILE *file, size_t *size);
 
 bool LCH_FileExists(const char *path);
@@ -23,4 +25,6 @@ bool LCH_FileDelete(const char *filename);
 
 bool LCH_FileCreateParentDirectories(const char *filename);
 
+LCH_List *LCH_FileListDirectory(const char *path, bool filter_hidden);
+
 #endif  // _LEECH_FILES_H

lib/instance.c

@@ -17,8 +17,9 @@ struct LCH_Instance {
   size_t major;
   size_t minor;
   size_t patch;
-  size_t max_chain_length;
+  size_t chain_length;
   bool pretty_print;
+  bool auto_purge;
   LCH_List *tables;
 };
 
@@ -72,25 +73,36 @@ LCH_Instance *LCH_InstanceLoad(const char *const work_dir) {
   }
 
   {
-    const LCH_Buffer *const key =
-        LCH_BufferStaticFromString("max_chain_length");
+    const LCH_Buffer *const key = LCH_BufferStaticFromString("chain_length");
     if (LCH_JsonObjectHasKey(config, key)) {
       double number;
       if (!LCH_JsonObjectGetNumber(config, key, &number)) {
         LCH_InstanceDestroy(instance);
         LCH_JsonDestroy(config);
         return NULL;
       }
-      if (!LCH_DoubleToSize(number, &(instance->max_chain_length))) {
+      if (!LCH_DoubleToSize(number, &(instance->chain_length))) {
         LCH_InstanceDestroy(instance);
         LCH_JsonDestroy(config);
         return NULL;
       }
     } else {
-      instance->max_chain_length = LCH_DEFAULT_MAX_CHAIN_LENGTH;
+      instance->chain_length = LCH_DEFAULT_PREFERED_CHAIN_LENGTH;
     }
-    LCH_LOG_DEBUG("config[\"max_chain_length\"] = \"%zu\"",
-                  instance->max_chain_length);
+    LCH_LOG_DEBUG("config[\"chain_length\"] = %zu", instance->chain_length);
+  }
+
+  {
+    instance->auto_purge = false;
+    const LCH_Buffer *const key = LCH_BufferStaticFromString("auto_purge");
+    if (LCH_JsonObjectHasKey(config, key)) {
+      const LCH_Json *const json = LCH_JsonObjectGet(config, key);
+      if (LCH_JsonIsTrue(json)) {
+        instance->auto_purge = true;
+      }
+    }
+    LCH_LOG_DEBUG("config[\"auto_purge\"] = %s",
+                  (instance->auto_purge) ? "true" : "false");
   }
 
   {
@@ -208,12 +220,17 @@ const char *LCH_InstanceGetWorkDirectory(const LCH_Instance *const self) {
   return self->work_dir;
 }
 
-size_t LCH_InstanceGetMaxChainLength(const LCH_Instance *const instance) {
+size_t LCH_InstanceGetPrefferedChainLength(const LCH_Instance *const instance) {
   assert(instance != NULL);
-  return instance->max_chain_length;
+  return instance->chain_length;
 }
 
-bool LCH_InstancePrettyPrint(const LCH_Instance *const instance) {
+bool LCH_InstanceShouldPrettyPrint(const LCH_Instance *const instance) {
   assert(instance != NULL);
   return instance->pretty_print;
 }
+
+bool LCH_InstanceShouldAutoPurge(const LCH_Instance *const instance) {
+  assert(instance != NULL);
+  return instance->auto_purge;
+}

lib/instance.h

@@ -42,8 +42,10 @@ const LCH_List *LCH_InstanceGetTables(const LCH_Instance *instance);
  */
 const char *LCH_InstanceGetWorkDirectory(const LCH_Instance *instance);
 
-size_t LCH_InstanceGetMaxChainLength(const LCH_Instance *instance);
+size_t LCH_InstanceGetPrefferedChainLength(const LCH_Instance *instance);
 
-bool LCH_InstancePrettyPrint(const LCH_Instance *instance);
+bool LCH_InstanceShouldPrettyPrint(const LCH_Instance *instance);
+
+bool LCH_InstanceShouldAutoPurge(const LCH_Instance *instance);
 
 #endif  // _LEECH_INSTANCE_H