Merge pull request #6 from atlanhq/cache-fail-open

Redis cache fail open handling
atlanhq · Jan 25, 2024 · 1f2b050 · 1f2b050
2 parents 9d45e3e + 5e75ff7
commit 1f2b050
Show file tree

Hide file tree

Showing 30 changed files with 82 additions and 53 deletions.
diff --git a/docs/configs/janusgraph-cfg.md b/docs/configs/janusgraph-cfg.md
@@ -19,6 +19,7 @@ Configuration options that modify JanusGraph's caching behavior
 | Name | Description | Datatype | Default Value | Mutability |
 | ---- | ---- | ---- | ---- | ---- |
 | cache.cache-keyspace-prefix | Set prefix for keyspace created in redis. | String | janusgraph | MASKABLE |
+| cache.cache-max-fail-open-count | Max queries to by-pass redis during redis fail over. If its set to 1000, after 1000 backend queries an attempt is made toquery redis again. This is to avoid waiting for every query during redis cluster failure. | Integer | 1000 | MASKABLE |
 | cache.cache-type | Enable or disable Redis cache (redis/inmemory) | String | inmemory | MASKABLE |
 | cache.db-cache | Whether to enable JanusGraph's database-level cache, which is shared across all transactions. Enabling this option speeds up traversals by holding hot graph elements in memory, but also increases the likelihood of reading stale data.  Disabling it forces each transaction to independently fetch graph elements from storage before reading/writing them. | Boolean | false | MASKABLE |
 | cache.db-cache-clean-wait | How long, in milliseconds, database-level cache will keep entries after flushing them.  This option is only useful on distributed storage backends that are capable of acknowledging writes without necessarily making them immediately visible. | Integer | 50 | GLOBAL_OFFLINE |

diff --git a/janusgraph-all/pom.xml b/janusgraph-all/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-all</artifactId>

diff --git a/janusgraph-backend-testutils/pom.xml b/janusgraph-backend-testutils/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-backend-testutils</artifactId>

diff --git a/janusgraph-berkeleyje/pom.xml b/janusgraph-berkeleyje/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-berkeleyje</artifactId>

diff --git a/janusgraph-bigtable/pom.xml b/janusgraph-bigtable/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-bigtable</artifactId>

diff --git a/janusgraph-core/pom.xml b/janusgraph-core/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-core</artifactId>

diff --git a/...c/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/ExpirationKCVSRedisCache.java b/...c/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/ExpirationKCVSRedisCache.java
@@ -15,7 +15,6 @@
 package org.janusgraph.diskstorage.keycolumnvalue.cache;
 
 import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheLoader;
 import org.janusgraph.core.JanusGraphException;
 import org.janusgraph.diskstorage.BackendException;
 import org.janusgraph.diskstorage.EntryList;
@@ -31,17 +30,19 @@
 import org.redisson.api.RLock;
 import org.redisson.api.RMapCache;
 import org.redisson.api.RedissonClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.CACHE_KEYSPACE_PREFIX;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.MAX_CACHE_FAIL_OPEN_COUNT;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_LOCK_LEASE_MS;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_CACHE_LOCK_WAIT_MS;
 import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REDIS_MAX_CACHE_SIZE;
@@ -52,11 +53,13 @@
 public class ExpirationKCVSRedisCache extends KCVSCache {
 
     public static final String REDIS_INDEX_CACHE_PREFIX = "index";
+    private final AtomicInteger CACHE_FAIL_OPEN_COUNT = new AtomicInteger(0);
+    private static int CACHE_MAX_FAIL_OPEN_COUNT = 100;
     private final long cacheTimeMS;
     private RedissonClient redissonClient;
     private RMapCache<KeySliceQuery, byte[]> redisCache;
     private RMapCache<StaticBuffer, ArrayList<KeySliceQuery>> redisIndexKeys;
-    private static Logger logger = Logger.getLogger("janusgraph-redis-logger");
+    private static final Logger log = LoggerFactory.getLogger("janusgraph-redis-logger");
     private static FSTConfiguration fastConf = FSTConfiguration.createDefaultConfiguration();
     private Configuration configuration;
 
@@ -68,39 +71,46 @@ public ExpirationKCVSRedisCache(final KeyColumnValueStore store, String metricsN
         Preconditions.checkArgument(System.currentTimeMillis() + 1000L * 3600 * 24 * 365 * 100 + cacheTimeMS > 0, "Cache expiration time too large, overflow may occur: %s", cacheTimeMS);
         this.cacheTimeMS = cacheTimeMS;
         Preconditions.checkArgument(invalidationGracePeriodMS >= 0, "Invalid expiration grace period: %s", invalidationGracePeriodMS);
-
+        CACHE_MAX_FAIL_OPEN_COUNT = configuration.get(MAX_CACHE_FAIL_OPEN_COUNT);
         redissonClient = RedissonCache.getRedissonClient(configuration);
         redisCache = redissonClient.getMapCache(String.join("-", configuration.get(CACHE_KEYSPACE_PREFIX), metricsName));
         redisIndexKeys = redissonClient.getMapCache(String.join("-", configuration.get(CACHE_KEYSPACE_PREFIX), REDIS_INDEX_CACHE_PREFIX, metricsName));
         redisCache.setMaxSize(configuration.get(REDIS_MAX_CACHE_SIZE), EvictionMode.LFU);
         redisIndexKeys.setMaxSize(configuration.get(REDIS_MAX_CACHE_SIZE), EvictionMode.LFU);
-        logger.info("********************** Configurations are loaded **********************");
+        log.info("********************** Cache configurations are loaded **********************");
     }
 
     @Override
     public EntryList getSlice(final KeySliceQuery query, final StoreTransaction txh) throws BackendException {
-        incActionBy(1, CacheMetricsAction.RETRIEVAL, txh);
-        try {
-            return get(query, () -> {
-                incActionBy(1, CacheMetricsAction.MISS, txh);
-                return store.getSlice(query, unwrapTx(txh));
-            });
-        } catch (Exception e) {
-            if (e instanceof JanusGraphException) throw (JanusGraphException) e;
-            else if (e.getCause() instanceof JanusGraphException) throw (JanusGraphException) e.getCause();
-            else throw new JanusGraphException(e);
+        if (CACHE_FAIL_OPEN_COUNT.get() <= 0) {
+            try {
+                return get(query, txh, () -> {
+                    incActionBy(1, CacheMetricsAction.MISS, txh);
+                    return store.getSlice(query, unwrapTx(txh));
+                });
+            } catch (Exception e) {
+                if (e instanceof JanusGraphException) {
+                    throw new JanusGraphException(e);
+                } else {
+                    CACHE_FAIL_OPEN_COUNT.set(CACHE_MAX_FAIL_OPEN_COUNT);
+                    incActionBy(1, CacheMetricsAction.ERROR, txh);
+                    log.warn("Exception occurred while fetching data from cache. query:{}", query.getKey(), e);
+                }
+            }
         }
+        /* fallback to backend store on cache failure until fail open */
+        CACHE_FAIL_OPEN_COUNT.decrementAndGet();
+        return store.getSlice(query, unwrapTx(txh));
     }
 
-    private EntryList get(KeySliceQuery query, Callable<EntryList> valueLoader) throws Exception {
+    private EntryList get(KeySliceQuery query, final StoreTransaction txh, Callable<EntryList> valueLoader) throws Exception {
         byte[] bytQuery = redisCache.get(query);
         EntryList entries = bytQuery != null ? (EntryList) fastConf.asObject(bytQuery) : null;
         if (entries == null) {
-            logger.log(Level.INFO, "Reading from the store.................");
             try {
                 entries = valueLoader.call();
                 if (entries == null) {
-                    throw new CacheLoader.InvalidCacheLoadException("valueLoader must not return null, key=" + query);
+                    throw new JanusGraphException("No value returned from backend data store to cache., key=" + query);
                 } else {
                     redisCache.fastPutAsync(query, fastConf.asByteArray(entries), this.cacheTimeMS, TimeUnit.MILLISECONDS);
                     RLock lock = redisIndexKeys.getLock(query.getKey());
@@ -112,16 +122,20 @@ private EntryList get(KeySliceQuery query, Callable<EntryList> valueLoader) thro
                         queryList.add(query);
                         redisIndexKeys.fastPutAsync(query.getKey(), queryList, this.cacheTimeMS, TimeUnit.MILLISECONDS);
                     } catch (InterruptedException e) {
-                        e.printStackTrace();
+                        log.warn("Exception during cache update operations", e);
                         throw e;
                     } finally {
                         lock.unlock();
+                        redisIndexKeys.removeAsync(query.getKey());
                     }
                 }
             } catch (Exception e) {
-                e.printStackTrace();
+                log.warn("Exception during get and set cache operations", e);
                 throw e;
             }
+        } else{
+            CACHE_FAIL_OPEN_COUNT.set(0);
+            incActionBy(1, CacheMetricsAction.RETRIEVAL, txh);
         }
         return entries;
     }
@@ -154,7 +168,7 @@ public Map<StaticBuffer, EntryList> getSlice(final List<StaticBuffer> keys, fina
                 if (subresult != null) {
                     results.put(key, subresult);
                     if (ksqs[i] != null) {
-                        logger.info("adding to cache subresult " + subresult);
+                        log.info("adding to cache subresult:{}", subresult);
                         redisCache.fastPutAsync(ksqs[i], fastConf.asByteArray(subresult), this.cacheTimeMS, TimeUnit.MILLISECONDS);
                         RLock lock = redisIndexKeys.getLock(ksqs[i].getKey());
                         try {
@@ -184,13 +198,22 @@ public void clearCache() {
 
     @Override
     public void invalidate(StaticBuffer key, List<CachableStaticBuffer> entries) {
+        if (CACHE_FAIL_OPEN_COUNT.get() > 0) {
+            CACHE_FAIL_OPEN_COUNT.decrementAndGet();
+            return;
+        }
         List<KeySliceQuery> keySliceQueryList = redisIndexKeys.get(key);
-        if (keySliceQueryList != null) {
-            for (KeySliceQuery keySliceQuery : keySliceQueryList) {
-                if (key.equals(keySliceQuery.getKey())) {
-                    redisCache.fastRemoveAsync(keySliceQuery);
+        try {
+            if (keySliceQueryList != null) {
+                for (KeySliceQuery keySliceQuery : keySliceQueryList) {
+                    if (key.equals(keySliceQuery.getKey())) {
+                        redisCache.fastRemoveAsync(keySliceQuery);
+                    }
                 }
             }
+        } catch (Exception e) {
+            CACHE_FAIL_OPEN_COUNT.set(CACHE_MAX_FAIL_OPEN_COUNT);
+            log.warn("Error occurred while invalidating cache for key:{}", key, e);
         }
     }
 

diff --git a/janusgraph-core/src/main/java/org/janusgraph/diskstorage/util/CacheMetricsAction.java b/janusgraph-core/src/main/java/org/janusgraph/diskstorage/util/CacheMetricsAction.java
@@ -19,7 +19,7 @@
 */
 public enum CacheMetricsAction {
 
-    RETRIEVAL("retrievals"), MISS("misses"), EXPIRE("expire");
+    RETRIEVAL("retrievals"), MISS("misses"), EXPIRE("expire"), ERROR("error");
 
     private final String name;
 

diff --git a/...h-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java b/...h-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java
@@ -370,6 +370,11 @@ public boolean apply(@Nullable String s) {
             "triggers eviction when set to 0).",
         ConfigOption.Type.MASKABLE, 120000L);
 
+    public static final ConfigOption<Integer> MAX_CACHE_FAIL_OPEN_COUNT = new ConfigOption<>(CACHE_NS,"cache-max-fail-open-count",
+        "Max queries to by-pass redis during redis fail over. If its set to 1000, after 1000 backend queries an attempt is made to" +
+            "query redis again. This is to avoid waiting for every query during redis cluster failure.",
+        ConfigOption.Type.MASKABLE, 1000);
+
     public static final ConfigOption<String> REDIS_CACHE_SENTINEL_URLS = new ConfigOption<>(CACHE_NS,"redis-cache-sentinel-urls",
         "csv values for multiple redis sentinel host:port urls.",
         ConfigOption.Type.MASKABLE, "localhost:26379");

diff --git a/janusgraph-cql/pom.xml b/janusgraph-cql/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
     </parent>
 
     <artifactId>janusgraph-cql</artifactId>

diff --git a/janusgraph-dist/pom.xml b/janusgraph-dist/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <packaging>pom</packaging>

diff --git a/janusgraph-doc/pom.xml b/janusgraph-doc/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <packaging>pom</packaging>

diff --git a/janusgraph-driver/pom.xml b/janusgraph-driver/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-driver</artifactId>

diff --git a/janusgraph-es/pom.xml b/janusgraph-es/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-es</artifactId>

diff --git a/janusgraph-examples/example-berkeleyje/pom.xml b/janusgraph-examples/example-berkeleyje/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-berkeleyje</artifactId>

diff --git a/janusgraph-examples/example-common/pom.xml b/janusgraph-examples/example-common/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-common</artifactId>

diff --git a/janusgraph-examples/example-cql/pom.xml b/janusgraph-examples/example-cql/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-cql</artifactId>

diff --git a/janusgraph-examples/example-hbase/pom.xml b/janusgraph-examples/example-hbase/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-hbase</artifactId>

diff --git a/janusgraph-examples/example-remotegraph/pom.xml b/janusgraph-examples/example-remotegraph/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-remotegraph</artifactId>

diff --git a/janusgraph-examples/example-tinkergraph/pom.xml b/janusgraph-examples/example-tinkergraph/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-tinkergraph</artifactId>

diff --git a/janusgraph-examples/pom.xml b/janusgraph-examples/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-examples</artifactId>

diff --git a/janusgraph-grpc/pom.xml b/janusgraph-grpc/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
     </parent>
     <artifactId>janusgraph-grpc</artifactId>
     <name>JanusGraph-gRPC: gRPC Components for JanusGraph</name>

diff --git a/janusgraph-hadoop/pom.xml b/janusgraph-hadoop/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-hadoop</artifactId>

diff --git a/janusgraph-hbase/pom.xml b/janusgraph-hbase/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
     </parent>
     <artifactId>janusgraph-hbase</artifactId>
     <name>JanusGraph-HBase: Distributed Graph Database</name>

diff --git a/janusgraph-inmemory/pom.xml b/janusgraph-inmemory/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-inmemory</artifactId>

diff --git a/janusgraph-lucene/pom.xml b/janusgraph-lucene/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.03</version>
+        <version>0.6.04</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-lucene</artifactId>