Added CMake build for (1) core (2) thrift (3) bench.

CMakeLists.txt

@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 2.8)
+project(succinct CXX)
+
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+if(COMPILER_SUPPORTS_CXX11)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+else()
+    CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
+    if(COMPILER_SUPPORTS_CXX0X)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
+    else()
+        message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
+    endif()
+endif()
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g")
+
+set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake-modules)
+
+add_subdirectory(core)
+add_subdirectory(bench)
+add_subdirectory(thrift)

bench/CMakeLists.txt

@@ -0,0 +1,36 @@
+cmake_minimum_required(VERSION 2.8)
+project(succinct-bench CXX)
+
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+if(COMPILER_SUPPORTS_CXX11)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+else()
+    CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
+    if(COMPILER_SUPPORTS_CXX0X)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
+    else()
+        message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
+    endif()
+endif()
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g")
+
+find_package(Thrift REQUIRED)
+
+set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
+
+file(MAKE_DIRECTORY ${EXECUTABLE_OUTPUT_PATH})
+
+set(INCLUDE include ../core/include)
+set(THRIFT_INCLUDE ../thrift/include)
+include_directories(${INCLUDE} ${THRIFT_INCLUDE})
+add_executable(fbench src/file-bench.cpp)
+add_executable(sbench src/shard-bench.cpp)
+add_executable(lsbench src/ls-bench.cpp)
+add_executable(surebench src/sure-bench.cpp)
+add_executable(ssbench src/succinctserver-bench.cpp)
+target_link_libraries(fbench succinct)
+target_link_libraries(sbench succinct)
+target_link_libraries(lsbench succinct)
+target_link_libraries(surebench succinct)
+target_link_libraries(ssbench sclient ${THRIFT_LIBRARIES})

benchmark/include/AdaptBenchmark.hpp → bench/include/AdaptBenchmark.hpp

@@ -7,8 +7,8 @@
 
 #include "Benchmark.hpp"
 #include "ZipfGenerator.hpp"
-#include "thrift/AdaptiveQueryService.h"
-#include "thrift/ports.h"
+#include "AdaptiveQueryService.h"
+#include "ports.h"
 
 #include <thread>
 #include <sstream>

benchmark/include/AdaptiveQueryServerBenchmark.hpp → bench/include/AdaptiveQueryServerBenchmark.hpp

@@ -7,8 +7,8 @@
 
 #include "Benchmark.hpp"
 #include "ZipfGenerator.hpp"
-#include "thrift/AdaptiveQueryService.h"
-#include "thrift/ports.h"
+#include "AdaptiveQueryService.h"
+#include "ports.h"
 
 #include <thread>
 #include <sstream>

benchmark/include/DynamicAdaptBenchmark.hpp → bench/include/DynamicAdaptBenchmark.hpp

@@ -7,13 +7,13 @@
 
 #include "Benchmark.hpp"
 #include "ZipfGenerator.hpp"
-#include "thrift/AdaptiveSuccinctService.h"
-#include "thrift/ports.h"
-#include "thrift/AdaptiveSuccinctClient.hpp"
-#include "thrift/AdaptiveManagementClient.hpp"
-#include "thrift/ResponseQueue.hpp"
+#include "AdaptiveSuccinctService.h"
+#include "ports.h"
+#include "AdaptiveSuccinctClient.hpp"
+#include "AdaptiveManagementClient.hpp"
+#include "ResponseQueue.hpp"
 
-#include "succinct/LayeredSuccinctShard.hpp"
+#include "LayeredSuccinctShard.hpp"
 
 #include <thread>
 #include <sstream>

benchmark/include/DynamicLoadBalancerBenchmark.hpp → bench/include/DynamicLoadBalancerBenchmark.hpp

@@ -7,9 +7,9 @@
 
 #include "Benchmark.hpp"
 #include "ZipfGenerator.hpp"
-#include "thrift/AdaptiveQueryService.h"
-#include "thrift/DynamicLoadBalancer.hpp"
-#include "thrift/ports.h"
+#include "AdaptiveQueryService.h"
+#include "DynamicLoadBalancer.hpp"
+#include "ports.h"
 
 #include <thread>
 #include <sstream>

benchmark/include/FileBenchmark.hpp → bench/include/FileBenchmark.hpp

@@ -2,7 +2,7 @@
 #define FILE_BENCHMARK_HPP
 
 #include "Benchmark.hpp"
-#include "succinct/SuccinctFile.hpp"
+#include "SuccinctFile.hpp"
 
 
 class FileBenchmark : public Benchmark {

bench/include/LayeredSuccinctShardBenchmark.hpp

@@ -0,0 +1,168 @@
+#ifndef LAYERED_SUCCINCT_SHARD_BENCHMARK_HPP
+#define LAYERED_SUCCINCT_SHARD_BENCHMARK_HPP
+
+#include "Benchmark.hpp"
+#include "ZipfGenerator.hpp"
+#include "LayeredSuccinctShard.hpp"
+
+#include <thread>
+#include <sstream>
+#include <unistd.h>
+
+class LayeredSuccinctShardBenchmark : public Benchmark {
+ private:
+  LayeredSuccinctShard *fd;
+  std::vector<int32_t> lengths;
+  std::string resfile;
+  double skew_keys;
+  double skew_lengths;
+
+  void generate_randoms() {
+    count_t q_cnt = fd->num_keys();
+    fprintf(stderr, "Generating zipf distribution with theta=%f, N=%lu...\n",
+            skew_keys, q_cnt);
+    ZipfGenerator z(skew_keys, q_cnt);
+    fprintf(stderr, "Generated zipf distribution, generating keys...\n");
+    for (count_t i = 0; i < 100000; i++) {
+      randoms.push_back(z.next());
+    }
+    fprintf(stderr, "Generated keys.\n");
+  }
+
+  void generate_lengths() {
+    count_t q_cnt = fd->num_keys();
+    int32_t min_len = 100;
+    int32_t max_len = 500;
+    fprintf(stderr, "Generating zipf distribution with theta=%f, N=%u...\n",
+            skew_lengths, (max_len - min_len));
+    ZipfGenerator z(skew_lengths, max_len - min_len);
+    fprintf(stderr, "Generated zipf distribution, generating lengths...\n");
+
+    for (count_t i = 0; i < 100000; i++) {
+      // Map zipf value to a length
+      int32_t len = z.next() + min_len;
+      assert(len >= min_len);
+      assert(len <= max_len);
+      lengths.push_back(len);
+    }
+    fprintf(stderr, "Generated lengths.\n");
+  }
+
+  void read_queries(std::string filename) {
+    std::ifstream inputfile(filename);
+    if (!inputfile.is_open()) {
+      fprintf(stderr, "Error: Query file [%s] may be missing.\n",
+              filename.c_str());
+      return;
+    }
+
+    std::string line, bin, query;
+    while (getline(inputfile, line)) {
+      // Extract key and value
+      int split_index = line.find_first_of('\t');
+      bin = line.substr(0, split_index);
+      query = line.substr(split_index + 1);
+      queries.push_back(query);
+    }
+    inputfile.close();
+  }
+
+ public:
+  LayeredSuccinctShardBenchmark(std::string filename, bool construct,
+                                uint32_t isa_sampling_rate,
+                                uint32_t sa_sampling_rate, std::string resfile,
+                                double skew, std::string queryfile = "")
+      : Benchmark() {
+
+    this->resfile = resfile;
+    this->skew_keys = skew;
+    this->skew_lengths = 1.0;  // Pure uniform for now
+    this->fd = new LayeredSuccinctShard(
+        0,
+        filename,
+        construct ?
+            SuccinctMode::CONSTRUCT_IN_MEMORY : SuccinctMode::LOAD_IN_MEMORY,
+        sa_sampling_rate, isa_sampling_rate);
+    if (construct) {
+      // Serialize and save to file
+      fd->serialize();
+    }
+
+    generate_randoms();
+    generate_lengths();
+
+    if (queryfile != "") {
+      read_queries(queryfile);
+    }
+  }
+
+  void measure_access_throughput(uint32_t len) {
+    fprintf(stderr, "Starting access throughput measurement...");
+    size_t storage_size = fd->storage_size();
+    std::string res;
+    std::ofstream res_stream(resfile + ".access",
+                             std::ofstream::out | std::ofstream::app);
+    uint64_t num_ops = 0;
+
+    time_t start_time = get_timestamp();
+    while (num_ops <= randoms.size()) {
+      fd->access(res, randoms[num_ops % randoms.size()], 0, len);
+      num_ops++;
+    }
+    time_t diff = get_timestamp() - start_time;
+    double thput = ((double) num_ops * 1000 * 1000) / ((double) diff);
+    res_stream << storage_size << "\t" << thput << "\n";
+    res_stream.close();
+    fprintf(stderr, "Done.\n");
+  }
+
+  void measure_get_throughput() {
+    fprintf(stderr, "Starting get throughput measurement...");
+    size_t storage_size = fd->storage_size();
+    std::string res;
+    std::ofstream res_stream(resfile + ".get",
+                             std::ofstream::out | std::ofstream::app);
+    uint64_t num_ops = 0;
+
+    time_t start_time = get_timestamp();
+    while (num_ops <= randoms.size()) {
+      fd->get(res, randoms[num_ops % randoms.size()]);
+      num_ops++;
+    }
+    time_t diff = get_timestamp() - start_time;
+    double thput = ((double) num_ops * 1000 * 1000) / ((double) diff);
+    res_stream << storage_size << "\t" << thput << "\n";
+    res_stream.close();
+    fprintf(stderr, "Done.\n");
+  }
+
+  void measure_search_throughput() {
+    fprintf(stderr, "Starting search throughput measurement...");
+    size_t storage_size = fd->storage_size();
+    std::set<int64_t> res;
+    std::ofstream res_stream(resfile + ".search",
+                             std::ofstream::out | std::ofstream::app);
+    uint64_t num_ops = 0;
+
+    time_t start_time = get_timestamp();
+    while (num_ops <= randoms.size()) {
+      fd->search(res, queries[num_ops % queries.size()]);
+      num_ops++;
+    }
+    time_t diff = get_timestamp() - start_time;
+    double thput = ((double) num_ops * 1000 * 1000) / ((double) diff);
+    res_stream << storage_size << "\t" << thput << "\n";
+    res_stream.close();
+    fprintf(stderr, "Done.\n");
+  }
+
+  void delete_layer(int32_t layer_id) {
+    if (layer_id >= 0) {
+      fprintf(stderr, "Deleting layer %d...\n", layer_id);
+      fd->remove_layer(layer_id);
+      fprintf(stderr, "Done.\n");
+    }
+  }
+};
+
+#endif

benchmark/include/ShardBenchmark.hpp → bench/include/ShardBenchmark.hpp

@@ -2,7 +2,7 @@
 #define SHARD_BENCHMARK_H
 
 #include "Benchmark.hpp"
-#include "succinct/SuccinctShard.hpp"
+#include "SuccinctShard.hpp"
 
 
 class ShardBenchmark :  public Benchmark {

benchmark/include/SuccinctServerBenchmark.hpp → bench/include/SuccinctServerBenchmark.hpp

@@ -6,9 +6,9 @@
 #include <thrift/transport/TBufferTransports.h>
 
 #include "Benchmark.hpp"
-#include "succinct/SuccinctShard.hpp"
-#include "thrift/SuccinctService.h"
-#include "thrift/ports.h"
+#include "SuccinctShard.hpp"
+#include "SuccinctService.h"
+#include "ports.h"
 
 using namespace ::apache::thrift;
 using namespace ::apache::thrift::protocol;

benchmark/src/file-bench.cpp → bench/src/file-bench.cpp

@@ -3,7 +3,7 @@
 #include <unistd.h>
 
 #include "FileBenchmark.hpp"
-#include "succinct/SuccinctFile.hpp"
+#include "SuccinctFile.hpp"
 
 void print_usage(char *exec) {
     fprintf(stderr, "Usage: %s [-m mode] [file]\n", exec);
@@ -54,7 +54,7 @@ int main(int argc, char **argv) {
         SuccinctFile fd(inputpath, SuccinctMode::LOAD_IN_MEMORY);
 
         // Benchmark core and file functions
-        FileBenchmark s_bench(fd, querypath);
+        FileBenchmark s_bench(&fd, querypath);
         s_bench.benchmark_core();
         s_bench.benchmark_file();
     } else {

benchmark/src/shard-bench.cpp → bench/src/shard-bench.cpp

@@ -2,11 +2,9 @@
 #include <fstream>
 #include <unistd.h>
 
-#include "succinct/SuccinctShard.hpp"
+#include "SuccinctShard.hpp"
 #include "ShardBenchmark.hpp"
 
-#include "thrift/DynamicLoadBalancer.hpp"
-
 void print_usage(char *exec) {
     fprintf(stderr, "Usage: %s [-m mode] [-s sa_sampling_rate] [-i isa_sampling_rate] [-x sampling_scheme] [-d delete-layers] [-c create-layers] [-n npa_sampling_rate] [-t type] [-l len] [-q queryfile] [file]\n", exec);
 }

benchmark/src/sure-bench.cpp → bench/src/sure-bench.cpp

@@ -4,8 +4,8 @@
 #include <time.h>
 #include <sys/time.h>
 
-#include "succinct/regex/RegEx.hpp"
-#include "succinct/SuccinctFile.hpp"
+#include "regex/RegEx.hpp"
+#include "SuccinctFile.hpp"
 
 // Debug
 void display(RegEx *re) {