Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding unit testing to paperless-ai #186

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6,602 changes: 4,898 additions & 1,704 deletions package-lock.json

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
"description": "AI based Tag, correspondent and meta data generation",
"main": "server.js",
"scripts": {
"test": "nodemon server.js"
"dev": "nodemon server.js",
"test": "jest"
},
"keywords": [
"paperless-ngx",
Expand Down Expand Up @@ -51,6 +52,7 @@
"eslint": "^9.17.0",
"eslint-config-prettier": "^9.1.0",
"globals": "^15.14.0",
"jest": "^29.7.0",
"prettier": "^3.4.2"
}
}
3 changes: 2 additions & 1 deletion routes/setup.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const express = require('express');
const router = express.Router();
const setupService = require('../services/setupService.js');
const SetupService = require('../services/setupService.js');
const setupService = new SetupService();
const paperlessService = require('../services/paperlessService.js');
const openaiService = require('../services/openaiService.js');
const ollamaService = require('../services/ollamaService.js');
Expand Down
3 changes: 2 additions & 1 deletion server.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ const config = require('./config/config');
const paperlessService = require('./services/paperlessService');
const AIServiceFactory = require('./services/aiServiceFactory');
const documentModel = require('./models/document');
const setupService = require('./services/setupService');
const SetupService = require('./services/setupService');
const setupService = new SetupService();
const setupRoutes = require('./routes/setup');
const cors = require('cors');
const cookieParser = require('cookie-parser');
Expand Down
24 changes: 24 additions & 0 deletions services/aiServiceFactory.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
const AiServiceFactory = require('./aiServiceFactory')
const {expect, test} = require('@jest/globals')
const config = require('../config/config')
const {OllamaService} = require("./ollamaService");
const {OpenAIService} = require("./openaiService");


test("if configured to be openai, returns an openaiService instance", () => {

config.aiProvider = "openai"

const service = AiServiceFactory.getService()
expect(service).toBeInstanceOf(OpenAIService)

})

test("if configured to be ollama, returns an ollamaService instance", () => {

config.aiProvider = "ollama"

const service = AiServiceFactory.getService()
expect(service).toBeInstanceOf(OllamaService)

})
3 changes: 2 additions & 1 deletion services/ollamaService.js
Original file line number Diff line number Diff line change
Expand Up @@ -362,4 +362,5 @@ class OllamaService {
}
}

module.exports = new OllamaService();
module.exports = new OllamaService();
module.exports.OllamaService = OllamaService;
3 changes: 2 additions & 1 deletion services/openaiService.js
Original file line number Diff line number Diff line change
Expand Up @@ -318,4 +318,5 @@ class OpenAIService {
}
}

module.exports = new OpenAIService();
module.exports = new OpenAIService();
module.exports.OpenAIService = OpenAIService;
51 changes: 30 additions & 21 deletions services/setupService.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,18 @@ const { OpenAI } = require('openai');
const config = require('../config/config');

class SetupService {
constructor() {
this.envPath = path.join(process.cwd(), 'data', '.env');
constructor(param = undefined) {
this.envPath = param?.envPath || path.join(process.cwd(), 'data', '.env');
this.configured = null; // Variable to store the configuration status
}

async loadConfig() {
try {
const envContent = await fs.readFile(this.envPath, 'utf8');
const config = {};
envContent.split('\n').forEach(line => {
const [key, value] = line.split('=');
if (key && value) {
config[key.trim()] = value.trim();
}
});
return config;
const envContent = {}

const result = require('dotenv').config({ path: this.envPath, processEnv: envContent });

return result.parsed
} catch (error) {
console.error('Error loading config:', error.message);
return null;
Expand Down Expand Up @@ -127,15 +123,7 @@ class SetupService {
// Ensure data directory exists
const dataDir = path.dirname(this.envPath);
await fs.mkdir(dataDir, { recursive: true });

const envContent = Object.entries(config)
.map(([key, value]) => {
if (key === "SYSTEM_PROMPT") {
return `${key}=\`${value}\n\``;
}
return `${key}=${value}`;
})
.join('\n');
const envContent = this.asEnvFileContent(config);

await fs.writeFile(this.envPath, envContent);

Expand All @@ -149,6 +137,27 @@ class SetupService {
}
}

/**
* Converts the provided `config` object and serializes the content in the dotenv format.
*
* This implementation respects multiline strings.
*
* @param config the key-value pairs to be written as a the body of an .env file
* @returns {string} the generated .env file content
*/
asEnvFileContent(config) {
return Object.entries(config)
.map(([key, value]) => {

if (typeof value === "string" && value.indexOf('\n') > 0) {
// multi line string. Encode using the multiline syntax
return `${key}="${value}"`;
}
return `${key}=${value}`;
})
.join('\n');
}

async isConfigured() {
if (this.configured !== null) {
return this.configured;
Expand Down Expand Up @@ -195,4 +204,4 @@ class SetupService {
}
}

module.exports = new SetupService();
module.exports = SetupService;
48 changes: 48 additions & 0 deletions services/setupService.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
const {expect, test} = require('@jest/globals')
const SetupService = require("./setupService");
const path = require("node:path");

test("parse a valid environment file with multiline contents", async () => {

const testfile = path.resolve(__dirname, "test-data", "setupService-multiline.env");

const service = new SetupService({envPath: testfile});

const config = await service.loadConfig();

expect(config).toHaveProperty("KEY1", "value1")
expect(config).toHaveProperty("KEY2", "value2")
expect(config).toHaveProperty("MULTILINE_KEY1", `This is line1
This is line2
This is line3`)
})

test("that no loaded configuration is populated in process.env", async () => {

const testfile = path.resolve(__dirname, "test-data", "setupService-multiline.env");

const service = new SetupService({envPath: testfile});

const config = await service.loadConfig();

expect(config).toHaveProperty("KEY1", "value1")
expect(process.env).not.toHaveProperty("KEY1")
})

test("writing env files with multiline strings", () => {

const service = new SetupService();

const envContent = service.asEnvFileContent({
"KEY1": "value1",
"MULTILINE_KEY1": `Line 1
Line 2
Line 3`
})

expect(envContent).toBe(`KEY1=value1
MULTILINE_KEY1="Line 1
Line 2
Line 3"`)

});
5 changes: 5 additions & 0 deletions services/test-data/setupService-multiline.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
KEY1=value1
KEY2=value2
MULTILINE_KEY1="This is line1
This is line2
This is line3"