XrdApps::JCache: by default now use the real remote namespace to stor…

…e in the JCache & allow to set the cache automatically to bypass for all applications listed under 'noapp' in the configuration file. This is useful for example for xrdcp where caching normally does not make sense.
cern-eos · Jul 24, 2024 · d54566a · d54566a
1 parent 91319d9
commit d54566a
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 9 deletions.
diff --git a/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.cc b/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.cc
@@ -24,6 +24,7 @@
 /*----------------------------------------------------------------------------*/
 #include "XrdClJCacheFile.hh"
 #include "file/CacheStats.hh"
+#include "file/Hierarchy.hh"
 /*----------------------------------------------------------------------------*/
 #include "XrdCl/XrdClMessageUtils.hh"
 /*----------------------------------------------------------------------------*/
@@ -35,6 +36,8 @@ bool XrdCl::JCacheFile::sEnableVectorCache = false;
 bool XrdCl::JCacheFile::sEnableSummary = true;
 bool XrdCl::JCacheFile::sEnableBypass = false;
 bool XrdCl::JCacheFile::sOpenAsync = false;
+bool XrdCl::JCacheFile::sFlatHierarchy = false;
+
 JCache::CacheStats XrdCl::JCacheFile::sStats(true);
 JCache::Cleaner XrdCl::JCacheFile::sCleaner;
 JournalManager XrdCl::JCacheFile::sJournalManager;
@@ -130,15 +133,30 @@ XRootDStatus JCacheFile::Open(const std::string &url, OpenFlags::Flags flags,
       mOpenState = OPENING;
       if (sEnableVectorCache || (sEnableJournalCache && !sEnableBypass)) {
         if ((flags & OpenFlags::Flags::Read) == OpenFlags::Flags::Read) {
-          std::string JournalDir =
-              sCachePath + "/" + VectorCache::computeSHA256(pUrl);
-          pJournalPath = JournalDir + "/journal";
-          // it can be that we cannot write the journal directory
-          if (!VectorCache::ensureLastSubdirectoryExists(JournalDir)) {
-            st = XRootDStatus(stError, errOSError);
-            std::cerr << "error: unable to create cache directory: "
-                      << JournalDir << std::endl;
-            return st;
+          if (sFlatHierarchy) {
+            // we use sha256 directory names in a flat hierarchy
+            std::string JournalDir =
+                sCachePath + "/" + VectorCache::computeSHA256(pUrl);
+            pJournalPath = JournalDir + "/journal";
+            // it can be that we cannot write the journal directory
+            if (!VectorCache::ensureLastSubdirectoryExists(JournalDir)) {
+              st = XRootDStatus(stError, errOSError);
+              std::cerr << "error: unable to create cache directory: "
+                        << JournalDir << std::endl;
+              return st;
+            }
+          } else {
+            // we keep the original path for the cache
+            XrdCl::URL url(pUrl);
+            std::string JournalDir =
+                sCachePath + url.GetPath();
+            pJournalPath = JournalDir + "/journal";
+            if (!JCache::makeHierarchy(pJournalPath)) {
+              st = XRootDStatus(stError, errOSError);
+              std::cerr << "error: unable to create cache directory: "
+                        << JournalDir << std::endl;
+              return st;
+            }
           }
         }
       }

diff --git a/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.hh b/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.hh
@@ -210,6 +210,9 @@ public:
   //----------------------------------------------------------------------------
   inline bool IsValid() { return true; }
 
+  //----------------------------------------------------------------------------
+  //! @brief set the local cache path
+  //! @param path Local cache path
   //----------------------------------------------------------------------------
   //! @brief set the local cache path and enable/disable journal/vector caches
   //! @param path Local cache path
@@ -225,6 +228,7 @@ public:
   static void SetBypass(const bool &value) { sEnableBypass = value; }
   static void SetSize(uint64_t size) { sCleaner.SetSize(size, sCachePath); }
   static void SetAsync(bool async) { sOpenAsync = async; }
+  static void SetFlatHierarchy(bool value) { sFlatHierarchy = value; }
 
   //----------------------------------------------------------------------------
   //! @brief static members pointing to cache settings
@@ -236,6 +240,8 @@ public:
   static bool sEnableBypass;
   static bool sEnableSummary;
   static bool sOpenAsync;
+  static bool sFlatHierarchy;
+
   static JournalManager sJournalManager;
 
   //! @brief set stats interval in seconds

diff --git a/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.hh b/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.hh
@@ -67,6 +67,7 @@ public:
       JCacheFile::SetAsync(ita != config->end()
                                ? (ita->second == "true") || (ita->second == "1")
                                : false);
+
       auto itb = config->find("bypass");
       JCacheFile::SetBypass(itb != config->end() ? (itb->second == "true") ||
                                                        (itb->second == "1")
@@ -132,6 +133,24 @@ public:
             (std::string(v).length()) ? std::stoll(std::string(v), 0, 10) : 0);
       }
 
+      Env *env = DefaultEnv::GetEnv();
+      std::string appName;
+      env->GetString("AppName", appName);
+      std::string noApps;
+      auto itapp = config->find("noapp");
+      if (itapp != config->end()) {
+        noApps = itapp->second;
+      }
+
+      if (const char *v = getenv("XRD_JCACHE_NOAPP")) {
+        noApps = v;
+      }
+
+      if (noApps.length() && (noApps.find(appName) != std::string::npos)) {
+        // switch the cache on bypass if we are in the 'noapp' entry list
+        JCacheFile::SetBypass(true);
+      }
+
       Log *log = DefaultEnv::GetLog();
       log->Info(1, "JCache : cache directory: %s",
                 JCacheFile::sCachePath.c_str());
@@ -145,6 +164,11 @@ public:
                 JCacheFile::sOpenAsync ? "true" : "false");
       log->Info(1, "JCache : bypass operation: %s",
                 JCacheFile::sEnableBypass ? "true" : "false");
+      log->Info(1, "JCache : running app: %s", appName.c_str());
+
+      if (noApps.length()) {
+        log->Info(1, "JCache : filtered apps: %s", noApps.c_str());
+      }
       if (JCacheFile::sJsonPath.length()) {
         log->Info(1, "JCache : json output to prefix: %s",
                   JCacheFile::sJsonPath.c_str());