Shared cache env-var tweaks (#1390)

1) Declares the wake shared cache no longer experimental and changes the env-var used for it to be WAKE_LOCAL_JOB_CACHE
2) Adds the ability to configure some config options via environment variables. In particular the user config can now be specified via an env-var which allows for greatly increased flexibility. I did however additionally allow env-vars for cache miss on failure, and cache size manipulation in case that's helpful but we anticipate only
3) I went ahead and added a command line override for user-config so that you can locally fix what might become a shell-global override of .wakeroot

README.md

@@ -174,6 +174,16 @@ Below is a full example
 }
 ```
 
+While there are many sources that a config option might come from, the following priority is always observed from lowest
+to highest:
+1) .wakeroot
+2) user config
+3) environment variables
+4) command line options
+
+So a command line option overides anything, an environment variable overrides user config and wakeroot, user config
+overrides wakeroot, and wakeroot overrides nothing
+
 # Documentation
 
 Documentation for wake can be found in [share/doc/wake](share/doc/wake).

share/wake/lib/system/job.wake

@@ -538,7 +538,7 @@ export def mkJobCacheRunner (hashFn: Result RunnerInput Error => Result String E
 
                 # Sometimes we want a read-only cache. For instance read-only pre-merge
                 # with read-write post-merge.
-                require None = getenv "WAKE_EXPERIMENTAL_JOB_CACHE_READ_ONLY"
+                require None = getenv "WAKE_LOCAL_JOB_CACHE_READ_ONLY"
                 else Pass ""
 
                 job_cache_add jobCacheAddJson
@@ -855,7 +855,7 @@ export def access (file: String) (mode: Integer): Boolean =
     (\f \m prim "access") file mode
 
 export def defaultRunner: Runner =
-    require Some _ = getenv "WAKE_EXPERIMENTAL_JOB_CACHE"
+    require Some _ = getenv "WAKE_LOCAL_JOB_CACHE"
     else fuseRunner
 
     # The fuseRunner does not actully mount over / and instead uses the

src/job_cache/job_cache.cpp

@@ -319,7 +319,7 @@ FindJobResponse Cache::read(const FindJobRequest &find_request) {
   for (int i = 0; i < 3; i++) {
     auto response = read_impl(find_request);
     if (response) {
-      wcl::log::info("Returning job response: cache_hit = %d", int(bool(response->match)));
+      wcl::log::info("Returning job response: cache_hit = %d", int(bool(response->match)))();
       return *response;
     }
 

src/runtime/config.cpp

@@ -335,9 +335,28 @@ bool WakeConfig::init(const std::string& wakeroot_path, const WakeConfigOverride
     }
   }
 
+  // The priority of config sources is the following from lowest priority to highest:
+  // 1) .wakeroot
+  // 2) user config
+  // 3) environment variables
+  // 4) command line options
+  //
+  // When parsing the user config, the user config path can't be in the user config
+  // but it can be anywhere else so before parsing the user config we parse wakeroot
+  // and then the other two sources. Once we parse the user config though we'll have
+  // go overwrite anything from the user config that should be form an env-var or
+  // a command line option.
+
   // Parse values from .wakeroot
   _config->set_all<WakeConfigProvenance::WakeRoot>(wakeroot_json);
 
+  // Sometimes we need to the user_config with an env-var so we check env-vars first here
+  _config->set_all_env_var();
+
+  // Furthermore users may choose to override the user config at the command line level so
+  // we run that to only to run it again later
+  _config->override_all(overrides);
+
   // Parse user config
   auto user_config_res = read_json_file(_config->user_config);
   if (!user_config_res) {
@@ -371,7 +390,12 @@ bool WakeConfig::init(const std::string& wakeroot_path, const WakeConfigOverride
   // Parse values from the user config
   _config->set_all<WakeConfigProvenance::UserConfig>(user_config_json);
 
-  // Finally apply command line overrides
+  // Set all env-vars again as they should override user configs. Note that
+  // this is the second time we set the env-vars.
+  _config->set_all_env_var();
+
+  // Finally apply command line overrides as they override everything. Note that
+  // this is the second time we set the overrides.
   _config->override_all(overrides);
 
   return true;

src/runtime/config.h

@@ -49,6 +49,9 @@ struct WakeConfigOverrides {
 
   // Determines if job cache should terminate on error or return a cache miss
   wcl::optional<bool> cache_miss_on_failure;
+
+  // Lets you specify an alternative user config
+  wcl::optional<std::string> user_config;
 };
 
 template <class T>
@@ -67,11 +70,13 @@ struct VersionPolicy {
   type version = "";
   static constexpr type VersionPolicy::*value = &VersionPolicy::version;
   static constexpr Override<input_type> override_value = nullptr;
+  static constexpr const char* env_var = nullptr;
 
   VersionPolicy() = default;
   static void set(VersionPolicy& p, const JAST& json);
   static void set_input(VersionPolicy& p, const input_type& v) { p.*value = v; }
   static void emit(const VersionPolicy& p, std::ostream& os) { os << p.*value; }
+  static void set_env_var(VersionPolicy& p, const char* env_var){};
 };
 
 struct UserConfigPolicy {
@@ -82,12 +87,14 @@ struct UserConfigPolicy {
   static constexpr bool allowed_in_userconfig = false;
   type user_config;
   static constexpr type UserConfigPolicy::*value = &UserConfigPolicy::user_config;
-  static constexpr Override<input_type> override_value = nullptr;
+  static constexpr Override<input_type> override_value = &WakeConfigOverrides::user_config;
+  static constexpr const char* env_var = "WAKE_USER_CONFIG";
 
   UserConfigPolicy();
   static void set(UserConfigPolicy& p, const JAST& json);
   static void set_input(UserConfigPolicy& p, const input_type& v) { p.*value = v; }
   static void emit(const UserConfigPolicy& p, std::ostream& os) { os << p.*value; }
+  static void set_env_var(UserConfigPolicy& p, const char* env_var) { p.*value = env_var; };
 };
 
 struct LogHeaderPolicy {
@@ -99,11 +106,13 @@ struct LogHeaderPolicy {
   type log_header = "[$stream] $source: ";
   static constexpr type LogHeaderPolicy::*value = &LogHeaderPolicy::log_header;
   static constexpr Override<input_type> override_value = &WakeConfigOverrides::log_header;
+  static constexpr const char* env_var = nullptr;
 
   LogHeaderPolicy() = default;
   static void set(LogHeaderPolicy& p, const JAST& json);
   static void set_input(LogHeaderPolicy& p, const input_type& v) { p.*value = v; }
   static void emit(const LogHeaderPolicy& p, std::ostream& os) { os << p.*value; }
+  static void set_env_var(LogHeaderPolicy& p, const char* env_var){};
 };
 
 struct LogHeaderSourceWidthPolicy {
@@ -117,11 +126,13 @@ struct LogHeaderSourceWidthPolicy {
       &LogHeaderSourceWidthPolicy::log_header_source_width;
   static constexpr Override<input_type> override_value =
       &WakeConfigOverrides::log_header_source_width;
+  static constexpr const char* env_var = nullptr;
 
   LogHeaderSourceWidthPolicy() {}
   static void set(LogHeaderSourceWidthPolicy& p, const JAST& json);
   static void set_input(LogHeaderSourceWidthPolicy& p, const input_type& v) { p.*value = v; }
   static void emit(const LogHeaderSourceWidthPolicy& p, std::ostream& os) { os << p.*value; }
+  static void set_env_var(LogHeaderSourceWidthPolicy& p, const char* env_var){};
 };
 
 struct LabelFilterPolicy {
@@ -133,6 +144,7 @@ struct LabelFilterPolicy {
   type label_filter;
   static constexpr type LabelFilterPolicy::*value = &LabelFilterPolicy::label_filter;
   static constexpr Override<input_type> override_value = &WakeConfigOverrides::label_filter;
+  static constexpr const char* env_var = nullptr;
 
   LabelFilterPolicy() : label_filter(std::make_unique<re2::RE2>(".*")) {}
 
@@ -145,6 +157,7 @@ struct LabelFilterPolicy {
   static void emit(const LabelFilterPolicy& p, std::ostream& os) {
     os << p.label_filter->pattern();
   }
+  static void set_env_var(LogHeaderSourceWidthPolicy& p, const char* value){};
 };
 
 struct SharedCacheMaxSize {
@@ -156,10 +169,14 @@ struct SharedCacheMaxSize {
   type max_cache_size = 25ULL << 30ULL;
   static constexpr type SharedCacheMaxSize::*value = &SharedCacheMaxSize::max_cache_size;
   static constexpr Override<input_type> override_value = &WakeConfigOverrides::max_cache_size;
+  static constexpr const char* env_var = "WAKE_SHARED_CACHE_MAX_SIZE";
 
   static void set(SharedCacheMaxSize& p, const JAST& json);
   static void set_input(SharedCacheMaxSize& p, const input_type& v) { p.*value = v; }
   static void emit(const SharedCacheMaxSize& p, std::ostream& os) { os << p.*value; }
+  static void set_env_var(SharedCacheMaxSize& p, const char* env_var) {
+    p.*value = std::stoull(env_var);
+  }
 };
 
 struct SharedCacheLowSize {
@@ -171,10 +188,14 @@ struct SharedCacheLowSize {
   type low_cache_size = 15ULL << 30ULL;
   static constexpr type SharedCacheLowSize::*value = &SharedCacheLowSize::low_cache_size;
   static constexpr Override<input_type> override_value = &WakeConfigOverrides::max_cache_size;
+  static constexpr const char* env_var = "WAKE_SHARED_CACHE_LOW_SIZE";
 
   static void set(SharedCacheLowSize& p, const JAST& json);
   static void set_input(SharedCacheLowSize& p, const input_type& v) { p.*value = v; }
   static void emit(const SharedCacheLowSize& p, std::ostream& os) { os << p.*value; }
+  static void set_env_var(SharedCacheLowSize& p, const char* env_var) {
+    p.*value = std::stoull(env_var);
+  }
 };
 
 struct SharedCacheMissOnFailure {
@@ -188,6 +209,7 @@ struct SharedCacheMissOnFailure {
       &SharedCacheMissOnFailure::cache_miss_on_failure;
   static constexpr Override<input_type> override_value =
       &WakeConfigOverrides::cache_miss_on_failure;
+  static constexpr const char* env_var = "WAKE_SHARED_CACHE_MISS_ON_FAILURE";
 
   static void set(SharedCacheMissOnFailure& p, const JAST& json);
   static void set_input(SharedCacheMissOnFailure& p, const input_type& v) { p.*value = v; }
@@ -198,6 +220,13 @@ struct SharedCacheMissOnFailure {
       os << "false";
     }
   }
+  static void set_env_var(SharedCacheMissOnFailure& p, const char* env_var) {
+    if (std::string(env_var) == "1") {
+      p.*value = true;
+    } else {
+      p.*value = false;
+    }
+  }
 };
 
 struct LogHeaderAlignPolicy {
@@ -209,6 +238,7 @@ struct LogHeaderAlignPolicy {
   type log_header_align = false;
   static constexpr type LogHeaderAlignPolicy::*value = &LogHeaderAlignPolicy::log_header_align;
   static constexpr Override<input_type> override_value = &WakeConfigOverrides::log_header_align;
+  static constexpr const char* env_var = nullptr;
 
   LogHeaderAlignPolicy() {}
   static void set(LogHeaderAlignPolicy& p, const JAST& json);
@@ -220,13 +250,14 @@ struct LogHeaderAlignPolicy {
       os << "false";
     }
   }
+  static void set_env_var(LogHeaderAlignPolicy& p, const char* env_var) {}
 };
 
 /********************************************************************
  * Generic WakeConfig implementation
  *********************************************************************/
 
-enum class WakeConfigProvenance { Default, WakeRoot, UserConfig, CommandLine };
+enum class WakeConfigProvenance { Default, WakeRoot, UserConfig, CommandLine, EnvVar };
 
 static inline const char* to_string(WakeConfigProvenance p) {
   switch (p) {
@@ -238,6 +269,8 @@ static inline const char* to_string(WakeConfigProvenance p) {
       return "UserConfig";
     case WakeConfigProvenance::CommandLine:
       return "Commandline";
+    case WakeConfigProvenance::EnvVar:
+      return "EnvVar";
   }
   return "Unknown (this is an error, please report this to Wake devs)";
 }
@@ -310,6 +343,17 @@ struct WakeConfigImpl : public Policies... {
     };
   }
 
+  template <class P>
+  auto set_env_var() {
+    return [this]() {
+      if (P::env_var == nullptr) return;
+      const char* env_var = getenv(P::env_var ? P::env_var : "");
+      if (env_var == nullptr) return;
+      provenance[P::key] = WakeConfigProvenance::EnvVar;
+      P::set_env_var(*this, env_var);
+    };
+  }
+
   template <class P>
   auto override_policy(const WakeConfigOverrides& overrides) {
     return [&overrides, this]() {
@@ -338,6 +382,8 @@ struct WakeConfigImpl : public Policies... {
     call_all(set_policy<p, Policies>(json)...);
   }
 
+  void set_all_env_var() { call_all(set_env_var<Policies>()...); }
+
   void override_all(const WakeConfigOverrides& overrides) {
     call_all(override_policy<Policies>(overrides)...);
   }

src/runtime/job.cpp

@@ -1771,7 +1771,7 @@ static PRIMFN(prim_job_cache_read) {
   // First, the user may have not turned on the job cache
   if (!internal_job_cache) {
     std::string s =
-        "A job cache has not been specified. Please use WAKE_JOB_CACHE=<path> to turn on job "
+        "A job cache has not been specified. Please use WAKE_LOCAL_JOB_CACHE=<path> to turn on job "
         "caching";
     size_t need = String::reserve(s.size()) + reserve_result();
     runtime.heap.reserve(need);
@@ -1826,7 +1826,7 @@ static PRIMFN(prim_job_cache_add) {
   // First, the user may have not turned on the job cache
   if (!internal_job_cache) {
     std::string s =
-        "A job cache has not been specified. Please use WAKE_JOB_CACHE=<path> to turn on job "
+        "A job cache has not been specified. Please use WAKE_LOCAL_JOB_CACHE=<path> to turn on job "
         "caching";
     size_t need = String::reserve(s.size()) + reserve_result();
     runtime.heap.reserve(need);

tests/config/nominal/pass.sh

@@ -6,4 +6,4 @@ if [ $(uname) != Linux ] ; then
 fi
 
 WAKE="${1:+$1/wake}"
-"${WAKE:-wake}" --config
+WAKE_SHARED_CACHE_MAX_SIZE=1024 "${WAKE:-wake}" --config

tests/config/nominal/stdout

@@ -4,7 +4,7 @@ Wake config:
   log_header = 'foobar $source: ' (WakeRoot)
   log_header_source_width = '13' (WakeRoot)
   label_filter = '.*' (Default)
-  max_cache_size = '1024' (WakeRoot)
+  max_cache_size = '1024' (EnvVar)
   low_cache_size = '512' (WakeRoot)
   cache_miss_on_failure = 'true' (WakeRoot)
   log_header_align = 'true' (WakeRoot)

tests/job-cache/basic-fetch/pass.sh

@@ -12,11 +12,11 @@ rm wake.db || true
 rm -rf .cache-hit || true
 rm -rf .cache-misses || true
 rm -rf .job-cache || true
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test
 
 rm wake.db
 rm -rf .cache-misses
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test
 if [ -z "$(ls -A .cache-hit)" ]; then
   echo "No cache hit found"
   exit 1

tests/job-cache/basic-lru/pass.sh

@@ -14,18 +14,18 @@ rm one.txt 2> /dev/null || true
 rm two.txt 2> /dev/null || true
 rm three.txt 2> /dev/null || true
 rm four.txt 2> /dev/null || true
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
 rm wake.db
 rm -rf .cache-misses
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test two
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test two
 rm wake.db
 rm -rf .cache-misses
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test three
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test three
 rm wake.db
 rm -rf .cache-misses
 
 # We should still be under the limit here. This is to ensure we mark test one as used
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
 rm wake.db
 rm -rf .cache-misses
 if [ -z "$(ls -A .cache-hit)" ]; then
@@ -34,13 +34,13 @@ if [ -z "$(ls -A .cache-hit)" ]; then
 fi
 
 # Now we're going to go over. Hopefully dropping two and three, but keeping one and four
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test four
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test four
 rm wake.db
 rm -rf .cache-misses
 
 # Now make sure we still get a hit on 1
 rm -rf .cache-hit  2> /dev/null || true
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
 rm wake.db
 if [ -z "$(ls -A .cache-hit)" ]; then
   echo "No cache hit found"
@@ -53,7 +53,7 @@ fi
 
 # And check four as well
 rm -rf .cache-hit
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test four
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test four
 rm wake.db
 if [ -z "$(ls -A .cache-hit)" ]; then
   echo "No cache hit found"
@@ -65,7 +65,7 @@ if [ -d ".cache-misses" ]; then
 fi
 
 # And check that we get misses on four and three
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test two
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test two
 rm wake.db
 if [ -z "$(ls -A .cache-misses)" ]; then
   echo "Expected a chache miss!!"
@@ -74,7 +74,7 @@ fi
 rm -rf .cache-misses
 
 
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_EXPERIMENTAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test three
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test three
 rm wake.db
 if [ -z "$(ls -A .cache-misses)" ]; then
   echo "Expected a chache miss!!"