Automatic import from remote drive (#86)

* Add initial files for handling automatic importing from seedboxes * Initial structure and storage driver integration * Add method to find files recursively * Remove . from file extensions * Add sample config for seedboxes * Filter which movies need to be import from remote storage * Basic importing of movies from remote storage * Add support for importing episodes * Update package-lock.json * Update package-lock.json * Add rest endpoint to import media from remote storage * Only import matched episodes and store episodes in series directories * Add option to disable an auto importer config
robinp7720 · Feb 11, 2022 · b689a7b · b689a7b
1 parent bfe5736
commit b689a7b
Show file tree

Hide file tree

Showing 19 changed files with 3,648 additions and 6,528 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -34,6 +34,7 @@
     "fluent-ffmpeg": "^2.1.2",
     "guessit-exec": "^0.0.1",
     "guessit-wrapper": "^1.0.1",
+    "jsftp": "^2.1.3",
     "jsonwebtoken": "^8.5.1",
     "mime-types": "^2.1.30",
     "mkdirp": "^1.0.4",
@@ -47,7 +48,7 @@
     "restify-cors-middleware": "^1.1.1",
     "restify-errors": "^8.0.2",
     "sequelize": "^6.6.2",
-    "sharp": "^0.28.1",
+    "sharp": "^0.29.1",
     "socket.io": "^2.3.0",
     "sqlite3": "^5.0.2",
     "uuid": "^8.3.2",
@@ -61,9 +62,9 @@
     "@types/bcrypt": "^3.0.1",
     "@types/restify-errors": "^4.3.3",
     "@types/socket.io": "^2.1.13",
+    "chai": "^4.3.3",
     "eslint": "^7.25.0",
     "eslint-plugin-jsdoc": "^32.3.3",
-    "chai": "^4.3.3",
     "expect.js": "^0.3.1",
     "jshint": "^2.12.0",
     "mocha": "^8.3.2",

diff --git a/res/config.json b/res/config.json
@@ -81,12 +81,12 @@
   },
   "fileExtensions": {
     "video": [
-      ".mp4",
-      ".avi",
-      ".iso",
-      ".m4v",
-      ".mkv",
-      ".mk3d"
+      "mp4",
+      "avi",
+      "iso",
+      "m4v",
+      "mkv",
+      "mk3d"
     ]
   },
   "authentication": {
@@ -126,5 +126,24 @@
         "key": "/etc/oblecto/keys/tria.pub"
       }
     }
-  }
+  },
+  "seedboxes": [
+    {
+      "name": "Main seedbox",
+      "storageDriver": "ftp",
+      "storageDriverOptions": {
+        "host": "",
+        "port": 21,
+        "username": "username",
+        "password": "password"
+      },
+      "mediaImport": {
+        "movieDirectory": "/downloads/finished/movie/",
+        "seriesDirectory": "/downloads/finished/series/"
+      },
+      "automaticImport": true,
+      "deleteOnImport": false,
+      "enabled": false
+    }
+  ]
 }
diff --git a/src/lib/artwork/movies/MovieArtworkCollector.js b/src/lib/artwork/movies/MovieArtworkCollector.js
@@ -1,5 +1,6 @@
 import { Movie } from '../../../models/movie';
 import { fileExists } from '../../../submodules/utils';
+import logger from '../../../submodules/logger';
 
 /**
  * @typedef {import('../../oblecto').default} Oblecto
@@ -20,7 +21,7 @@ export default class MovieArtworkCollector {
      * @returns {Promise<void>}
      */
     async collectArtworkMovieFanart(movie) {
-        if (!await fileExists(this.oblecto.artworkUtils.movieFanartPath(movie))) return;
+        if (await fileExists(this.oblecto.artworkUtils.movieFanartPath(movie))) return;
 
         this.oblecto.queue.queueJob('downloadMovieFanart', movie);
     }
@@ -31,7 +32,7 @@ export default class MovieArtworkCollector {
      * @returns {Promise<void>}
      */
     async collectArtworkMoviePoster(movie) {
-        if (!await fileExists(this.oblecto.artworkUtils.moviePosterPath(movie))) return;
+        if (await fileExists(this.oblecto.artworkUtils.moviePosterPath(movie))) return;
 
         this.oblecto.queue.queueJob('downloadMoviePoster', movie);
     }
@@ -41,6 +42,8 @@ export default class MovieArtworkCollector {
      * @returns {Promise<void>}
      */
     async collectAllMovieFanart() {
+        logger.log('DEBUG', 'Collecting Movie fanart to download');
+
         let movies = await Movie.findAll();
 
         for (let movie of movies) {
@@ -53,6 +56,8 @@ export default class MovieArtworkCollector {
      * @returns {Promise<void>}
      */
     async collectAllMoviePosters() {
+        logger.log('DEBUG', 'Collecting Movie posters to download');
+
         let movies = await Movie.findAll();
 
         for (let movie of movies) {

diff --git a/src/lib/indexers/files/FileIndexer.js b/src/lib/indexers/files/FileIndexer.js
@@ -20,6 +20,11 @@ export default class FileIndexer {
         this.oblecto.queue.registerJob('indexFileStreams', this.indexVideoFileStreams);
     }
 
+    /**
+     *
+     * @param videoPath
+     * @return {File}
+     */
     async indexVideoFile(videoPath) {
         let parsedPath = Path.parse(videoPath);
 

diff --git a/src/lib/indexers/movies/MovieCollector.js b/src/lib/indexers/movies/MovieCollector.js
@@ -33,7 +33,7 @@ export default class MovieCollector {
      * @returns {Promise<void>}
      */
     async collectFile(file) {
-        let extension = path.parse(file).ext.toLowerCase();
+        let extension = path.parse(file).ext.toLowerCase().replace('.', '');
 
         if (this.oblecto.config.fileExtensions.video.indexOf(extension) !== -1) {
             this.oblecto.queue.queueJob('indexMovie',{ path: file });

diff --git a/src/lib/indexers/series/SeriesCollector.js b/src/lib/indexers/series/SeriesCollector.js
@@ -1,5 +1,5 @@
 import recursive from 'recursive-readdir';
-import path from 'path';
+import { extname } from 'path';
 
 /**
  * @typedef {import('../../oblecto').default} Oblecto
@@ -41,7 +41,7 @@ export default class SeriesCollector {
      * @returns {Promise<void>}
      */
     async collectFile(file) {
-        let extension = path.parse(file).ext.toLowerCase();
+        let extension = extname(file).toLowerCase().replace('.','');
 
         if (this.oblecto.config.fileExtensions.video.indexOf(extension) !== -1) {
             this.oblecto.queue.queueJob('indexEpisode',{ path: file });

diff --git a/src/lib/indexers/series/SeriesIndexer.js b/src/lib/indexers/series/SeriesIndexer.js
@@ -1,4 +1,4 @@
-import { Op } from 'sequelize';
+import {Op} from 'sequelize';
 
 import AggregateIdentifier from '../../common/AggregateIdentifier';
 
@@ -7,16 +7,17 @@ import TmdbEpisodeIdentifier from './identifiers/TmdbEpisodeIdentifier';
 import TvdbSeriesIdentifier from './identifiers/TvdbSeriesIdentifier';
 import TvdbEpisodeIdentifier from './identifiers/TvdbEpisodeIdentifier';
 
-import { Series } from '../../../models/series';
-import { Episode } from '../../../models/episode';
-import { File } from '../../../models/file';
+import {Series} from '../../../models/series';
+import {Episode} from '../../../models/episode';
+import {File} from '../../../models/file';
 
 import IdentificationError from '../../errors/IdentificationError';
 import logger from '../../../submodules/logger';
 import guessit from '../../../submodules/guessit';
 
 /**
  * @typedef {import('../../oblecto').default} Oblecto
+ * @typedef {import('../../../submodules/guessit').GuessitIdentification} GuessitIdentification
  */
 
 /**
@@ -98,6 +99,14 @@ export default class SeriesIndexer {
         return series;
     }
 
+    async identify(episodePath) {
+        const guessitIdentification = await guessit.identify(episodePath);
+        const seriesIdentification = await this.seriesIdentifier.identify(episodePath, guessitIdentification);
+        const episodeIdentification = await this.episodeIdentifer.identify(episodePath, guessitIdentification, seriesIdentification);
+
+        return { ...seriesIdentification, ...episodeIdentification };
+    }
+
     /**
      * Index a specific file and identify it as a series
      *
@@ -112,8 +121,6 @@ export default class SeriesIndexer {
          */
         const guessitIdentification = await guessit.identify(episodePath);
 
-        console.log(guessitIdentification.source);
-
         // Some single season shows usually don't have a season in the title,
         // therefore whe should set it to 1 by default.
         if (!guessitIdentification.season) {

diff --git a/src/lib/oblecto/index.js b/src/lib/oblecto/index.js
@@ -42,6 +42,7 @@ import FileIndexer from '../indexers/files/FileIndexer';
 
 import { initDatabase } from '../../submodules/database';
 import StreamSessionController from '../streamSessions/StreamSessionController';
+import SeedboxController from '../seedbox/SeedboxController';
 
 export default class Oblecto {
     constructor(config) {
@@ -88,6 +89,9 @@ export default class Oblecto {
 
         this.streamSessionController = new StreamSessionController(this);
 
+        this.seedboxController = new SeedboxController(this);
+        this.seedboxController.loadAllSeedboxes();
+
         if (config.federation.enable) {
             this.fedartionController = new FederationController(this);
             this.federationClientController = new FederationClientController(this);

diff --git a/src/lib/queue/index.js b/src/lib/queue/index.js
@@ -13,7 +13,7 @@ export default class Queue {
             if (!this.jobs[job.id]) return callback();
 
             let jobTimeout = setTimeout(() => {
-                logger.log('WARN', `Job ${job.id} is taking a long time. Maybe something is wrong?`, JSON.stringify(job));
+                logger.log('DEBUG', `Job ${job.id} is taking a long time. Maybe something is wrong?`, JSON.stringify(job));
             }, 20000);
 
             this.jobs[job.id](job.attr)

diff --git a/src/lib/seedbox/Seedbox.js b/src/lib/seedbox/Seedbox.js
@@ -0,0 +1,54 @@
+import { extname, normalize as normalizePath } from 'path';
+import SeedboxImportFTP from './SeedboxImportDrivers/SeedboxImportFTP';
+import WarnExtendableError from '../errors/WarnExtendableError';
+import logger from '../../submodules/logger';
+import index from 'async';
+
+export default class Seedbox {
+    constructor(seedboxConfig) {
+        this.moviePath = normalizePath(seedboxConfig.mediaImport.movieDirectory);
+        this.seriesPath = normalizePath(seedboxConfig.mediaImport.seriesDirectory);
+        this.name = seedboxConfig.name;
+
+        this.initStorageDriver(seedboxConfig.storageDriver, seedboxConfig.storageDriverOptions);
+    }
+
+    initStorageDriver(seedboxStorageDriver, seedboxStorageDriverOptions) {
+        switch (seedboxStorageDriver.toLowerCase()) {
+            case 'ftp':
+                this.storageDriver = new SeedboxImportFTP(seedboxStorageDriverOptions);
+                return;
+        }
+
+        return new WarnExtendableError('Invalid seedbox storage driver');
+    }
+
+    async findAll(indexPath, fileTypes) {
+        logger.log('DEBUG', `Finding files in ${indexPath}`);
+
+        const toIndex = [];
+        const indexed = [];
+
+        toIndex.push(indexPath);
+
+        while (toIndex.length > 0) {
+            let current = toIndex.pop();
+
+            const entries = await this.storageDriver.list(current);
+
+            for (const entry of entries) {
+                switch (entry.type) {
+                    case 0: // Type 0 is a file
+                        if (fileTypes.indexOf(extname(`${current}/${entry.name}`).replace('.','')) !== -1)
+                            indexed.push(normalizePath(`${current}/${entry.name}`));
+                        break;
+                    case 1: // Type 1 is a directory
+                        toIndex.push(normalizePath(`${current}/${entry.name}`));
+                        break;
+                }
+            }
+        }
+
+        return indexed;
+    }
+}