From 828b67967bcbc0d17e49c2a4ab5d01ee7e6feaac Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 8 Jul 2024 09:52:01 +0700 Subject: [PATCH] fix: #644 - model start attach mode can just work once (#851) --- .../infrastructure/commanders/init.command.ts | 4 +- .../commanders/models/model-pull.command.ts | 6 +- .../commanders/models/model-start.command.ts | 57 +++++++++++++++---- .../commanders/shortcuts/run.command.ts | 8 +-- .../commanders/usecases/init.cli.usecases.ts | 8 ++- .../usecases/models.cli.usecases.ts | 4 +- .../file-manager/file-manager.service.ts | 8 +++ .../src/usecases/cortex/cortex.usecases.ts | 17 ++---- .../src/usecases/models/models.usecases.ts | 1 + cortex-js/src/utils/logs.ts | 6 +- 10 files changed, 77 insertions(+), 42 deletions(-) diff --git a/cortex-js/src/infrastructure/commanders/init.command.ts b/cortex-js/src/infrastructure/commanders/init.command.ts index 30a80143c..e9999d562 100644 --- a/cortex-js/src/infrastructure/commanders/init.command.ts +++ b/cortex-js/src/infrastructure/commanders/init.command.ts @@ -36,9 +36,7 @@ export class InitCommand extends CommandRunner { async run(passedParams: string[], options?: InitOptions): Promise { if (options?.silent) { - const installationOptions = - await this.initUsecases.defaultInstallationOptions(); - await this.initUsecases.installEngine(installationOptions); + await this.initUsecases.installEngine(undefined); } else { options = await this.inquirerService.ask( 'init-run-mode-questions', diff --git a/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts b/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts index 93dfbcc7a..a37ab8632 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts @@ -60,11 +60,7 @@ export class ModelPullCommand extends CommandRunner { !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine)) ) { console.log('\n'); - await this.initUsecases.installEngine( - await this.initUsecases.defaultInstallationOptions(), - 'latest', - engine, - ); + await this.initUsecases.installEngine(undefined, 'latest', engine); } this.telemetryUsecases.sendEvent( [ diff --git a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts index 63c7130c9..fd4d0b140 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts @@ -11,7 +11,7 @@ import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { InitCliUsecases } from '../usecases/init.cli.usecases'; -import { existsSync } from 'node:fs'; +import { createReadStream, existsSync, statSync, watchFile } from 'node:fs'; import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; import { join } from 'node:path'; import { Engines } from '../types/engine.interface'; @@ -61,7 +61,9 @@ export class ModelStartCommand extends CommandRunner { !Array.isArray(existingModel.files) || /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0]) ) { - checkingSpinner.fail(`Model ${modelId} not found on filesystem.\nPlease try 'cortex pull ${modelId}' first.`); + checkingSpinner.fail( + `Model ${modelId} not found on filesystem.\nPlease try 'cortex pull ${modelId}' first.`, + ); process.exit(1); } @@ -74,18 +76,19 @@ export class ModelStartCommand extends CommandRunner { !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine)) ) { const engineSpinner = ora('Installing engine...').start(); - await this.initUsecases.installEngine( - await this.initUsecases.defaultInstallationOptions(), - 'latest', - engine, - ); + await this.initUsecases.installEngine(undefined, 'latest', engine); engineSpinner.succeed(); } + + // Attached - stdout logs + if (options.attach) { + this.attachLogWatch(); + } + await this.cortexUsecases - .startCortex(options.attach) + .startCortex() .then(() => this.modelsCliUsecases.startModel(modelId, options.preset)) - .then(console.log) - .then(() => !options.attach && process.exit(0)); + .then(() => options.attach && ora('Model is running...').start()); } modelInquiry = async () => { @@ -120,4 +123,38 @@ export class ModelStartCommand extends CommandRunner { parseTemplate(value: string) { return value; } + + /** + * Attach to the log file and watch for changes + */ + private async attachLogWatch() { + const logPath = await this.fileService.getLogPath(); + const initialSize = statSync(logPath).size; + const logStream = createReadStream(logPath, { + start: initialSize, + encoding: 'utf-8', + autoClose: false, + }); + logStream.on('data', (chunk) => { + console.log(chunk); + }); + watchFile(logPath, (curr, prev) => { + // Check if the file size has increased + if (curr.size > prev.size) { + // Calculate the position to start reading from + const position = prev.size; + + // Create a new read stream from the updated position + const updateStream = createReadStream(logPath, { + encoding: 'utf8', + start: position, + }); + + // Read the newly written content + updateStream.on('data', (chunk) => { + console.log(chunk); + }); + } + }); + } } diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index 41b6c9abe..146785eef 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -88,16 +88,12 @@ export class RunCommand extends CommandRunner { !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine)) ) { const engineSpinner = ora('Installing engine...').start(); - await this.initUsecases.installEngine( - await this.initUsecases.defaultInstallationOptions(), - 'latest', - engine, - ); + await this.initUsecases.installEngine(undefined, 'latest', engine); engineSpinner.succeed('Engine installed'); } return this.cortexUsecases - .startCortex(false) + .startCortex() .then(() => this.modelsCliUsecases.startModel(modelId, options.preset)) .then(() => this.chatCliUsecases.chat(modelId, options.threadId)); } diff --git a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts index 819aaa566..d5d041f2c 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts @@ -51,11 +51,15 @@ export class InitCliUsecases { * @param version */ installEngine = async ( - options: InitOptions, + options?: InitOptions, version: string = 'latest', engine: string = 'default', force: boolean = true, ): Promise => { + // Use default option if not defined + if (!options) { + options = await this.defaultInstallationOptions(); + } const configs = await this.fileManagerService.getConfig(); if (configs.initialized && !force) return; @@ -271,7 +275,7 @@ export class InitCliUsecases { private detectInstructions = (): Promise< 'AVX' | 'AVX2' | 'AVX512' | undefined > => { - const cpuInstruction = cpuInfo.cpuInfo()[0]?? 'AVX' + const cpuInstruction = cpuInfo.cpuInfo()[0] ?? 'AVX'; console.log(cpuInstruction, 'CPU instructions detected'); return Promise.resolve(cpuInstruction); }; diff --git a/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts index cb9bc4e15..e43391fe6 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts @@ -41,11 +41,11 @@ export class ModelsCliUsecases { .catch(async (e) => { console.error('Model start failed with reason:', e.message); - printLastErrorLines(await this.fileService.getDataFolderPath(), 5); + printLastErrorLines(await this.fileService.getLogPath()); console.log( 'For more information, please check the logs at: %s', - join(await this.fileService.getDataFolderPath(), 'cortex.log'), + await this.fileService.getLogPath(), ); process.exit(1); }); diff --git a/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts b/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts index a1d7a0735..7bfbd1e5e 100644 --- a/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts +++ b/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts @@ -249,6 +249,14 @@ export class FileManagerService { return join(await this.getDataFolderPath(), 'cortex-cpp', 'engines'); } + /** + * Get log path + * @returns the path to the cortex engines folder + */ + async getLogPath(): Promise { + return join(await this.getDataFolderPath(), 'cortex.log'); + } + async createFolderIfNotExistInDataFolder(folderName: string): Promise { const dataFolderPath = await this.getDataFolderPath(); const folderPath = join(dataFolderPath, folderName); diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts index cb8aef592..e4df5d098 100644 --- a/cortex-js/src/usecases/cortex/cortex.usecases.ts +++ b/cortex-js/src/usecases/cortex/cortex.usecases.ts @@ -29,9 +29,7 @@ export class CortexUsecases { * @param attach * @returns */ - async startCortex( - attach: boolean = false, - ): Promise { + async startCortex(): Promise { const configs = await this.fileManagerService.getConfig(); const host = configs.cortexCppHost; const port = configs.cortexCppPort; @@ -56,14 +54,11 @@ export class CortexUsecases { 'cortex-cpp', ); - const writer = openSync( - join(await this.fileManagerService.getDataFolderPath(), 'cortex.log'), - 'a+', - ); + const writer = openSync(await this.fileManagerService.getLogPath(), 'a+'); // go up one level to get the binary folder, have to also work on windows this.cortexProcess = spawn(cortexCppPath, args, { - detached: !attach, + detached: true, cwd: cortexCppFolderPath, stdio: [0, writer, writer], env: { @@ -143,9 +138,9 @@ export class CortexUsecases { /** * Check whether the Cortex CPP is healthy - * @param host - * @param port - * @returns + * @param host + * @param port + * @returns */ healthCheck(host: string, port: number): Promise { return fetch(CORTEX_CPP_HEALTH_Z_URL(host, port)) diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index 6d714799d..471ad47b8 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -227,6 +227,7 @@ export class ModelsUsecases { }; this.eventEmitter.emit('model.event', modelEvent); if (e.code === AxiosError.ERR_BAD_REQUEST) { + loadingModelSpinner.succeed('Model loaded'); return { message: 'Model already loaded', modelId, diff --git a/cortex-js/src/utils/logs.ts b/cortex-js/src/utils/logs.ts index 085be60a9..8cd77f23d 100644 --- a/cortex-js/src/utils/logs.ts +++ b/cortex-js/src/utils/logs.ts @@ -8,12 +8,12 @@ import { createInterface } from 'readline'; * @param numLines */ export async function printLastErrorLines( - dataFolderPath: string, - numLines: number = 5, + logPath: string, + numLines: number = 10, ): Promise { const errorLines: string[] = []; - const fileStream = createReadStream(join(dataFolderPath, 'cortex.log')); + const fileStream = createReadStream(logPath); const rl = createInterface({ input: fileStream, crlfDelay: Infinity,