[yugabyte#15481] DocDB: Remove some unused parts of RocksDB

Summary:
Removing unused RocksDB features:
- HDFS-related code
- RocksDB Redis support (different from YugabyteDB's Redis support -- we are not using RocksDB's Redis-related code)
- RocksDB TTL support (again, we are not using that, YugabyteDB's TTL implementation is completely different)
- "xfunc" tests (they have always been disabled and don't work if enabled)
- NodeJS bindings for RocksDB
- env_win.cc

Also revert the change to build_postgres.py introduced in debbf5a ( D21824 ) that disabled rsyncing src/postgres to the build directory in case the Postgres build stamp did not change. The code sync is useful during development in case when only Postgres *.sql or *.out regression test files have changed, and those files are excluded from computing the Postgres build stamp.

Test Plan: Jenkins

Reviewers: timur, sergei

Reviewed By: sergei

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D21940

python/yb/build_postgres.py

@@ -890,6 +890,14 @@ def build_postgres(self) -> None:
             self.clean_postgres()
 
         mkdir_p(self.pg_build_root)
+        if self.should_configure:
+            # Regardless of build stamp, the postgres code in src should be synced to the code in
+            # build.  It's fine to do this only for the configure step because the make step depends
+            # on the configure step as defined in src/postgres/CMakeLists.txt.
+            with WorkDirContext(self.pg_build_root):
+                self.sync_postgres_source()
+
+        self.set_env_vars('configure')
         saved_build_stamp = self.get_saved_build_stamp()
         initial_build_stamp = self.get_build_stamp(include_env_vars=True)
         initial_build_stamp_no_env = self.get_build_stamp(include_env_vars=False)
@@ -907,15 +915,6 @@ def build_postgres(self) -> None:
                     self.pg_build_root, self.steps_description())
                 return
 
-        if self.should_configure:
-            # Regardless of build stamp, the postgres code in src should be synced to the code in
-            # build.  It's fine to do this only for the configure step because the make step depends
-            # on the configure step as defined in src/postgres/CMakeLists.txt.
-            with WorkDirContext(self.pg_build_root):
-                self.sync_postgres_source()
-
-        self.set_env_vars('configure')
-
         with WorkDirContext(self.pg_build_root):
             if self.should_configure:
                 configure_start_time_sec = time.time()

src/yb/rocksdb/CMakeLists.txt

@@ -75,7 +75,6 @@ set(ROCKSDB_SRCS
     db/write_batch_base.cc
     db/write_controller.cc
     db/write_thread.cc
-    db/xfunc_test_points.cc
     db/db_iterator_wrapper.cc
     memtable/hash_linklist_rep.cc
     memtable/hash_skiplist_rep.cc
@@ -123,7 +122,6 @@ set(ROCKSDB_SRCS
     util/delete_scheduler.cc
     util/dynamic_bloom.cc
     util/env.cc
-    util/env_hdfs.cc
     util/env_posix.cc
     util/io_posix.cc
     util/thread_posix.cc
@@ -146,9 +144,7 @@ set(ROCKSDB_SRCS
     utilities/merge_operators/string_append/stringappend.cc
     utilities/merge_operators/uint64add.cc
     utilities/options/options_util.cc
-    utilities/redis/redis_lists.cc
     utilities/table_properties_collectors/compact_on_deletion_collector.cc
-    utilities/ttl/db_ttl_impl.cc
     utilities/write_batch_with_index/write_batch_with_index.cc
     utilities/write_batch_with_index/write_batch_with_index_internal.cc
     util/event_logger.cc
@@ -282,11 +278,8 @@ if (BUILD_TESTS AND
   target_compile_options(slice_transform_test PRIVATE "-Wno-array-bounds")
 endif()
 
-ADD_YB_TEST(utilities/merge_operators/string_append/stringappend_test)
 ADD_YB_TEST(utilities/options/options_util_test)
-ADD_YB_TEST(utilities/redis/redis_lists_test)
 ADD_YB_TEST(utilities/table_properties_collectors/compact_on_deletion_collector_test)
-ADD_YB_TEST(utilities/ttl/ttl_test)
 ADD_YB_TEST(utilities/write_batch_with_index/write_batch_with_index_test)
 
 if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")

src/yb/rocksdb/db/column_family.cc

@@ -45,7 +45,6 @@
 #include "yb/rocksdb/util/compression.h"
 #include "yb/rocksdb/util/options_helper.h"
 #include "yb/rocksdb/util/statistics.h"
-#include "yb/rocksdb/util/xfunc.h"
 
 #include "yb/util/logging.h"
 #include <glog/logging.h>
@@ -195,14 +194,6 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
   if (result.max_write_buffer_number_to_maintain < 0) {
     result.max_write_buffer_number_to_maintain = result.max_write_buffer_number;
   }
-  XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions",
-             xf_transaction_set_memtable_history1,
-             xf_transaction_set_memtable_history,
-             &result.max_write_buffer_number_to_maintain);
-  XFUNC_TEST("memtablelist_history_clear", "transaction_xftest_SanitizeOptions",
-             xf_transaction_clear_memtable_history1,
-             xf_transaction_clear_memtable_history,
-             &result.max_write_buffer_number_to_maintain);
 
   if (!result.prefix_extractor) {
     DCHECK(result.memtable_factory);

src/yb/rocksdb/db/db_bloom_filter_test.cc

@@ -1137,8 +1137,6 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
 }  // namespace
 
 TEST_F(DBBloomFilterTest, PrefixScan) {
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
-      kSkipNoPrefix);
   while (ChangeFilterOptions()) {
     int count;
     Slice prefix;
@@ -1186,7 +1184,6 @@ TEST_F(DBBloomFilterTest, PrefixScan) {
     ASSERT_EQ(env_->random_read_counter_.Read(), 2);
     Close();
   }  // end of while
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
 }
 
 TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {

src/yb/rocksdb/db/db_impl.cc

@@ -56,7 +56,6 @@
 #include "yb/util/priority_thread_pool.h"
 #include "yb/util/atomic.h"
 
-#include "yb/rocksdb/db/auto_roll_logger.h"
 #include "yb/rocksdb/db/builder.h"
 #include "yb/rocksdb/db/compaction_job.h"
 #include "yb/rocksdb/db/compaction_picker.h"
@@ -114,7 +113,6 @@
 #include "yb/rocksdb/util/stop_watch.h"
 #include "yb/rocksdb/util/sync_point.h"
 #include "yb/rocksdb/util/task_metrics.h"
-#include "yb/rocksdb/util/xfunc.h"
 #include "yb/rocksdb/db/db_iterator_wrapper.h"
 
 #include "yb/util/status_log.h"
@@ -4679,9 +4677,6 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
   auto cfh = down_cast<ColumnFamilyHandleImpl*>(column_family);
   auto cfd = cfh->cfd();
 
-  XFUNC_TEST("", "managed_new", managed_new1, xf_manage_new,
-             reinterpret_cast<DBImpl*>(this),
-             const_cast<ReadOptions*>(&read_options), is_snapshot_supported_);
   if (read_options.managed) {
     if ((read_options.tailing) || (read_options.snapshot != nullptr) ||
         (is_snapshot_supported_)) {
@@ -4780,9 +4775,6 @@ Status DBImpl::NewIterators(
   }
   iterators->clear();
   iterators->reserve(column_families.size());
-  XFUNC_TEST("", "managed_new", managed_new1, xf_manage_new,
-             reinterpret_cast<DBImpl*>(this),
-             const_cast<ReadOptions*>(&read_options), is_snapshot_supported_);
   if (read_options.managed) {
     if ((!read_options.tailing) && (read_options.snapshot == nullptr) &&
         (!is_snapshot_supported_)) {
@@ -4911,16 +4903,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
   Status status;
 
-  bool xfunc_attempted_write = false;
-  XFUNC_TEST("transaction", "transaction_xftest_write_impl",
-             xf_transaction_write1, xf_transaction_write, write_options,
-             db_options_, my_batch, callback, this, &status,
-             &xfunc_attempted_write);
-  if (xfunc_attempted_write) {
-    // Test already did the write
-    return status;
-  }
-
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
   WriteThread::Writer w;
   w.batch = my_batch;

src/yb/rocksdb/db/db_tailing_iter_test.cc

@@ -309,8 +309,6 @@ TEST_F(DBTestTailingIterator, TailingIteratorDeletes) {
 }
 
 TEST_F(DBTestTailingIterator, TailingIteratorPrefixSeek) {
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
-             kSkipNoPrefix);
   ReadOptions read_options;
   read_options.tailing = true;
 
@@ -340,7 +338,6 @@ TEST_F(DBTestTailingIterator, TailingIteratorPrefixSeek) {
 
   iter->Next();
   ASSERT_TRUE(!iter->Valid());
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
 }
 
 TEST_F(DBTestTailingIterator, TailingIteratorIncomplete) {
@@ -575,8 +572,6 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorDeletes) {
 }
 
 TEST_F(DBTestTailingIterator, ManagedTailingIteratorPrefixSeek) {
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
-             kSkipNoPrefix);
   ReadOptions read_options;
   read_options.tailing = true;
   read_options.managed = true;
@@ -607,7 +602,6 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorPrefixSeek) {
 
   iter->Next();
   ASSERT_TRUE(!iter->Valid());
-  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
 }
 
 TEST_F(DBTestTailingIterator, ManagedTailingIteratorIncomplete) {

src/yb/rocksdb/db/db_test.cc

@@ -821,7 +821,6 @@ TEST_F(DBTest, GetFromVersions) {
 
 TEST_F(DBTest, GetSnapshot) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
   do {
     CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
     // Try with both a short key and a long key
@@ -1337,7 +1336,6 @@ TEST_F(DBTest, IterMulti) {
 // by using reseek rather than sequential scan
 TEST_F(DBTest, IterReseek) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
   Options options = CurrentOptions(options_override);
   options.max_sequential_skip_in_iterations = 3;
   options.create_if_missing = true;
@@ -1519,7 +1517,6 @@ TEST_F(DBTest, IterPrevMaxSkip) {
 
 TEST_F(DBTest, IterWithSnapshot) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
   do {
     CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
     ASSERT_OK(Put(1, "key1", "val1"));
@@ -2608,7 +2605,7 @@ TEST_F(DBTest, IteratorPinsRef) {
 
 TEST_F(DBTest, Snapshot) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
+
   do {
     CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
     ASSERT_OK(Put(0, "foo", "0v1"));
@@ -2672,7 +2669,7 @@ TEST_F(DBTest, Snapshot) {
 
 TEST_F(DBTest, HiddenValuesAreRemoved) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
+
   do {
     Options options = CurrentOptions(options_override);
     CreateAndReopenWithCF({"pikachu"}, options);
@@ -2709,7 +2706,7 @@ TEST_F(DBTest, HiddenValuesAreRemoved) {
 
 TEST_F(DBTest, CompactBetweenSnapshots) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
+
   do {
     Options options = CurrentOptions(options_override);
     options.disable_auto_compactions = true;
@@ -2771,7 +2768,7 @@ TEST_F(DBTest, UnremovableSingleDelete) {
   // Because a subsequent SingleDelete(A) would delete the Put(A, v2)
   // but not Put(A, v1), so Get(A) would return v1.
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
+
   do {
     Options options = CurrentOptions(options_override);
     options.disable_auto_compactions = true;
@@ -3470,7 +3467,7 @@ TEST_F(DBTest, SnapshotFiles) {
 
 TEST_F(DBTest, CompactOnFlush) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
+
   do {
     Options options = CurrentOptions(options_override);
     options.disable_auto_compactions = true;
@@ -4330,7 +4327,7 @@ class MultiThreadedDBTest : public DBTest,
 
 TEST_P(MultiThreadedDBTest, MultiThreaded) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
+
   std::vector<std::string> cfs;
   for (int i = 1; i < kColumnFamilies; ++i) {
     cfs.push_back(ToString(i));
@@ -4863,7 +4860,7 @@ INSTANTIATE_TEST_CASE_P(
 
 TEST_P(DBTestRandomized, Randomized) {
   anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
+
   Options options = CurrentOptions(options_override);
   DestroyAndReopen(options);
 

src/yb/rocksdb/db/db_test_util.cc

@@ -253,9 +253,6 @@ Options DBHolder::CurrentOptions(
     const anon::OptionsOverride& options_override) {
   // this redundant copy is to minimize code change w/o having lint error.
   Options options = defaultOptions;
-  XFUNC_TEST("", "dbtest_options", inplace_options1, GetXFTestOptions,
-             reinterpret_cast<Options*>(&options),
-             options_override.skip_policy);
   BlockBasedTableOptions table_options;
   bool set_block_based_table_factory = true;
   switch (option_config_) {

src/yb/rocksdb/db/db_test_util.h

@@ -74,7 +74,6 @@
 #endif  // !(defined NDEBUG) || !defined(OS_WIN)
 #include "yb/rocksdb/util/testharness.h"
 #include "yb/rocksdb/util/testutil.h"
-#include "yb/rocksdb/util/xfunc.h"
 #include "yb/rocksdb/utilities/merge_operators.h"
 
 #include "yb/util/test_util.h" // For ASSERT_OK
@@ -183,9 +182,6 @@ class AtomicCounter {
 
 struct OptionsOverride {
   std::shared_ptr<const FilterPolicy> filter_policy = nullptr;
-
-  // Used as a bit mask of individual enums in which to skip an XF test point
-  int skip_policy = 0;
 };
 
 }  // namespace anon