Merge pull request #149 from autonomys/update-persistent-cache

Update persistent file cache

.gitignore

@@ -37,5 +37,6 @@ coverage/
 *.tmp
 *.temp
 
+.cache/
 database.sqlite
 .cursorrules

backend/__tests__/e2e/uploads/files.spec.ts

@@ -29,7 +29,6 @@ import {
   nodesRepository,
 } from '../../../src/repositories/index.js'
 import { InteractionType } from '../../../src/models/objects/interactions.js'
-import { databaseDownloadCache } from '../../../src/services/download/databaseDownloadCache/index.js'
 import { memoryDownloadCache } from '../../../src/services/download/memoryDownloadCache/index.js'
 import {
   OwnerRole,
@@ -38,6 +37,7 @@ import {
 import { FileGateway } from '../../../src/services/dsn/fileGateway/index.js'
 import { jest } from '@jest/globals'
 import { downloadService } from '../../../src/services/download/index.js'
+import { fsCache } from '../../../src/services/download/fsCache/singleton.js'
 import { handleArchivedObjects } from '../../../src/services/upload/nodeRemover/index.js'
 
 const files = [
@@ -223,10 +223,10 @@ files.map((file, index) => {
       })
 
       it('download cache should be updated', async () => {
-        const asyncFromDatabase = await databaseDownloadCache.get(cid)
+        const asyncFromDatabase = await fsCache.get(cid)
         expect(asyncFromDatabase).not.toBeNull()
         const fileArrayFromDatabase = await asyncIterableToPromiseOfArray(
-          asyncFromDatabase!,
+          asyncFromDatabase!.data,
         )
         const fileBufferFromDatabase = Buffer.concat(fileArrayFromDatabase)
         expect(fileBufferFromDatabase).toEqual(rndBuffer)
@@ -323,11 +323,11 @@ files.map((file, index) => {
         expect(metadata?.is_archived).toBe(true)
 
         expect(memoryDownloadCache.has(cid)).toBe(true)
-        expect(await databaseDownloadCache.has(cid)).toBe(true)
+        expect(fsCache.get(cid)).not.toBeNull()
       })
 
-      it('should be able to downloaded from gateway', async () => {
-        await databaseDownloadCache.clear()
+      it('should be able to remove the nodes', async () => {
+        await fsCache.clear()
         await memoryDownloadCache.clear()
 
         const downloadFileMock = jest

backend/migrations/20250127141651-delete-download-cache-schema.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var dbm;
+var type;
+var seed;
+var fs = require('fs');
+var path = require('path');
+var Promise;
+
+/**
+  * We receive the dbmigrate dependency from dbmigrate initially.
+  * This enables us to not have to rely on NODE_PATH.
+  */
+exports.setup = function(options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+  Promise = options.Promise;
+};
+
+exports.up = function(db) {
+  var filePath = path.join(__dirname, 'sqls', '20250127141651-delete-download-cache-schema-up.sql');
+  return new Promise( function( resolve, reject ) {
+    fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
+      if (err) return reject(err);
+      console.log('received data: ' + data);
+
+      resolve(data);
+    });
+  })
+  .then(function(data) {
+    return db.runSql(data);
+  });
+};
+
+exports.down = function(db) {
+  var filePath = path.join(__dirname, 'sqls', '20250127141651-delete-download-cache-schema-down.sql');
+  return new Promise( function( resolve, reject ) {
+    fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
+      if (err) return reject(err);
+      console.log('received data: ' + data);
+
+      resolve(data);
+    });
+  })
+  .then(function(data) {
+    return db.runSql(data);
+  });
+};
+
+exports._meta = {
+  "version": 1
+};

backend/migrations/sqls/20250127141651-delete-download-cache-schema-down.sql

@@ -0,0 +1,14 @@
+CREATE SCHEMA IF NOT EXISTS download_cache;
+
+CREATE TABLE IF NOT EXISTS download_cache.file_parts (
+    "cid" TEXT,
+    "index" INTEGER NOT NULL,
+    "data" BYTEA NOT NULL,
+    PRIMARY KEY (cid, index)
+);
+
+CREATE TABLE IF NOT EXISTS download_cache.registry (
+    "cid" TEXT PRIMARY KEY,
+    "last_accessed_at" TIMESTAMP WITH TIME ZONE NOT NULL,
+    "size" BIGINT NOT NULL
+);

backend/migrations/sqls/20250127141651-delete-download-cache-schema-up.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS download_cache.file_parts;
+DROP TABLE IF EXISTS download_cache.registry;
+DROP SCHEMA IF EXISTS download_cache;

backend/package.json

@@ -15,22 +15,27 @@
   "dependencies": {
     "@autonomys/auto-dag-data": "^1.0.8",
     "@autonomys/auto-drive": "^1.0.5",
+    "@keyvhq/sqlite": "^2.1.6",
     "@polkadot/api": "^12.3.1",
     "@polkadot/types": "^13.0.1",
     "@polkadot/util-crypto": "^13.0.2",
     "aws-sdk": "^2.1692.0",
     "body-parser": "^1.20.2",
+    "cache-manager": "^6.4.0",
     "cors": "^2.8.5",
     "db-migrate": "^0.11.14",
     "db-migrate-pg": "^1.5.2",
     "dotenv": "^16.4.5",
     "express": "^4.19.2",
     "jsonwebtoken": "^9.0.2",
+    "keyv": "^5.2.1",
+    "lru-cache": "^11.0.2",
     "multer": "^1.4.5-lts.1",
     "multiformats": "^13.2.2",
     "pg": "^8.13.0",
     "pg-format": "^1.0.4",
     "pizzip": "^3.1.7",
+    "tar": "^7.4.3",
     "uuid": "^10.0.0",
     "websocket": "^1.0.35",
     "winston": "^3.17.0",

backend/src/config.ts

@@ -1,8 +1,10 @@
 import { env } from './utils/misc.js'
 
-const DEFAULT_CHUNK_SIZE = 10 * 1024 ** 2
 const DEFAULT_MAX_CACHE_SIZE = BigInt(10 * 1024 ** 3)
 
+const DEFAULT_CACHE_MAX_SIZE = 10 * 1024 ** 3 // 10GB
+const DEFAULT_CACHE_TTL = 1000000 // 1000000 seconds
+
 export const config = {
   logLevel: env('LOG_LEVEL', 'info'),
   postgres: {
@@ -12,17 +14,6 @@ export const config = {
   requestSizeLimit: env('REQUEST_SIZE_LIMIT', '200mb'),
   corsAllowedOrigins: process.env.CORS_ALLOWED_ORIGINS,
   rpcEndpoint: env('RPC_ENDPOINT', 'ws://localhost:9944'),
-  databaseDownloadCache: {
-    chunkSize: Number(
-      env('DATABASE_DOWNLOAD_CACHE_CHUNK_SIZE', DEFAULT_CHUNK_SIZE.toString()),
-    ),
-    maxCacheSize: BigInt(
-      env(
-        'DATABASE_DOWNLOAD_CACHE_MAX_SIZE',
-        DEFAULT_MAX_CACHE_SIZE.toString(),
-      ),
-    ),
-  },
   memoryDownloadCache: {
     maxCacheSize: Number(
       env('MEMORY_DOWNLOAD_CACHE_MAX_SIZE', DEFAULT_MAX_CACHE_SIZE.toString()),
@@ -36,4 +27,9 @@ export const config = {
     url: env('AUTH_SERVICE_URL', 'http://localhost:3030'),
     token: env('AUTH_SERVICE_API_KEY'),
   },
+  cacheDir: env('CACHE_DIR', './.cache'),
+  cacheMaxSize: Number(
+    env('CACHE_MAX_SIZE', DEFAULT_CACHE_MAX_SIZE.toString()),
+  ),
+  cacheTtl: Number(env('CACHE_TTL', DEFAULT_CACHE_TTL.toString())),
 }