diff --git a/opencti-platform/opencti-graphql/config/default.json b/opencti-platform/opencti-graphql/config/default.json index ff3d30f67557..5445de451cfc 100644 --- a/opencti-platform/opencti-graphql/config/default.json +++ b/opencti-platform/opencti-graphql/config/default.json @@ -7,6 +7,7 @@ "enabled": true, "enabled_ui": true, "enabled_dev_features": [], + "enable_logs_safe_shutdown": false, "https_cert": { "ca": [], "key": null, @@ -20,6 +21,12 @@ "logs_console": true, "logs_max_files": 7, "logs_directory": "./logs", + "logs_shipping": false, + "logs_shipping_level": "info", + "logs_shipping_env_var_prefix": "APP__", + "logs_graylog_host": "127.0.0.1", + "logs_graylog_port": 12201, + "logs_graylog_adapter": "udp", "logs_redacted_inputs": ["password", "secret", "token"], "extended_error_message": false }, @@ -31,7 +38,13 @@ "logs_files": true, "logs_console": true, "logs_max_files": 7, - "logs_directory": "./logs" + "logs_directory": "./logs", + "logs_shipping": false, + "logs_shipping_level": "info", + "logs_shipping_env_var_prefix": "APP_AUDIT_", + "logs_graylog_host": "127.0.0.1", + "logs_graylog_port": 12201, + "logs_graylog_adapter": "udp" }, "event_loop_logs": { "enabled": false, diff --git a/opencti-platform/opencti-graphql/package.json b/opencti-platform/opencti-graphql/package.json index 2c2181346f30..045356141fd9 100644 --- a/opencti-platform/opencti-graphql/package.json +++ b/opencti-platform/opencti-graphql/package.json @@ -102,6 +102,7 @@ "express-session": "1.18.1", "fast-json-patch": "3.1.1", "file-type": "19.6.0", + "gelf-pro": "1.4.0", "github-api": "3.4.0", "graphql": "16.9.0", "graphql-constraint-directive": "5.4.3", @@ -159,6 +160,7 @@ "validator": "13.12.0", "winston": "3.17.0", "winston-daily-rotate-file": "5.0.0", + "winston-transport": "4.9.0", "ws": "8.18.0", "xml2js": "0.6.2" }, diff --git a/opencti-platform/opencti-graphql/src/boot.js b/opencti-platform/opencti-graphql/src/boot.js index b6b322ce131b..5221d033213c 100644 --- a/opencti-platform/opencti-graphql/src/boot.js +++ b/opencti-platform/opencti-graphql/src/boot.js @@ -1,4 +1,4 @@ -import { environment, getStoppingState, logApp, setStoppingState } from './config/conf'; +import { environment, getStoppingState, logApp, setStoppingState, shutdownLoggers } from './config/conf'; import platformInit, { checkFeatureFlags, checkSystemDependencies } from './initialization'; import cacheManager from './manager/cacheManager'; import { shutdownRedisClients } from './database/redis'; @@ -38,6 +38,17 @@ export const platformStart = async () => { throw modulesError; } } catch (mainError) { + try { + await shutdownLoggers(); + } catch (e) { + /* + errors when shutting down the loggers can't be logged to them, so we just try using the standard console as a + "best effort" to give them some visibility + */ + // eslint-disable-next-line no-console + console.error(e); + } + process.exit(1); } }; diff --git a/opencti-platform/opencti-graphql/src/config/conf.js b/opencti-platform/opencti-graphql/src/config/conf.js index 547d6645082f..7d986f162de3 100644 --- a/opencti-platform/opencti-graphql/src/config/conf.js +++ b/opencti-platform/opencti-graphql/src/config/conf.js @@ -34,6 +34,7 @@ import { UNKNOWN_ERROR, UnknownError, UnsupportedError } from './errors'; import { ENTITY_TYPE_PUBLIC_DASHBOARD } from '../modules/publicDashboard/publicDashboard-types'; import { AI_BUS } from '../modules/ai/ai-types'; import { SUPPORT_BUS } from '../modules/support/support-types'; +import { createLogShippingTransport } from './log-shipping'; // https://golang.org/src/crypto/x509/root_linux.go const LINUX_CERTFILES = [ @@ -97,6 +98,7 @@ nconf.file('default', resolveEnvFile('default')); const appLogLevel = nconf.get('app:app_logs:logs_level'); const appLogFileTransport = booleanConf('app:app_logs:logs_files', true); const appLogConsoleTransport = booleanConf('app:app_logs:logs_console', true); +const appLogShippingTransport = booleanConf('app:app_logs:logs_shipping', false); export const appLogLevelMaxDepthSize = nconf.get('app:app_logs:control:max_depth_size') ?? 5; export const appLogLevelMaxDepthKeys = nconf.get('app:app_logs:control:max_depth_keys') ?? 30; export const appLogLevelMaxArraySize = nconf.get('app:app_logs:control:max_array_size') ?? 50; @@ -197,6 +199,10 @@ if (appLogFileTransport) { if (appLogConsoleTransport) { appLogTransports.push(new winston.transports.Console()); } +if (appLogShippingTransport) { + const conf = nconf.get('app:app_logs'); + appLogTransports.push(createLogShippingTransport(conf)); +} const appLogger = winston.createLogger({ level: appLogLevel, @@ -207,6 +213,7 @@ const appLogger = winston.createLogger({ // Setup audit log logApp const auditLogFileTransport = booleanConf('app:audit_logs:logs_files', true); const auditLogConsoleTransport = booleanConf('app:audit_logs:logs_console', true); +const auditLogShippingTransport = booleanConf('app:audit_logs:logs_shipping', false); const auditLogTransports = []; if (auditLogFileTransport) { const dirname = nconf.get('app:audit_logs:logs_directory'); @@ -222,6 +229,10 @@ if (auditLogFileTransport) { if (auditLogConsoleTransport) { auditLogTransports.push(new winston.transports.Console()); } +if (auditLogShippingTransport) { + const conf = nconf.get('app:audit_logs'); + auditLogTransports.push(createLogShippingTransport(conf)); +} const auditLogger = winston.createLogger({ level: 'info', format: format.combine(timestamp(), format.errors({ stack: true }), format.json()), @@ -311,6 +322,26 @@ export const logTelemetry = { } }; +export function shutdownLoggers() { + const safeShutdown = booleanConf('app:enable_logs_safe_shutdown', false); + + if (safeShutdown) { + const shutdownPromises = [appLogger, auditLogger, supportLogger].map( + (logger) => new Promise( + (resolve) => { + logger + .end() + .on('finish', resolve); + } + ) + ); + + return Promise.all(shutdownPromises); + } + + return Promise.resolve(); +} + const BasePathConfig = nconf.get('app:base_path')?.trim() ?? ''; const AppBasePath = BasePathConfig.endsWith('/') ? BasePathConfig.slice(0, -1) : BasePathConfig; export const basePath = isEmpty(AppBasePath) || AppBasePath.startsWith('/') ? AppBasePath : `/${AppBasePath}`; diff --git a/opencti-platform/opencti-graphql/src/config/gelf-transport.js b/opencti-platform/opencti-graphql/src/config/gelf-transport.js new file mode 100644 index 000000000000..d522052f810a --- /dev/null +++ b/opencti-platform/opencti-graphql/src/config/gelf-transport.js @@ -0,0 +1,54 @@ +/* eslint-disable */ + +/* + * This is a straight copy of https://github.com/fchristle/winston-gelf/blob/5420e52bc6a9830dc4a56494097c752fddcfcabc/index.js + * with a single change, noted below. + * We disable the eslint rules that would cause warnings in the original code. + */ + +const Transport = require('winston-transport'); +const logger = require('gelf-pro'); + +const levels = { + emerg: 'emergency', + alert: 'alert', + crit: 'critical', + error: 'error', + warn: 'warn', + notice: 'notice', + info: 'info', + debug: 'debug', +}; + +class GelfTransport extends Transport { + constructor(opts) { + super(opts); + this.logger = Object.create(logger); + this.logger.setConfig(opts.gelfPro); + } + + log({ level, message, ...extra }, callback) { + setImmediate(() => { + this.emit('logged', { level, message, extra }); + }); + + if (typeof extra === 'object') { + for (const key in extra) { + const value = extra[key]; + if (value instanceof Error) { + extra = value; + } + } + } + + const graylogLevel = levels[level] || levels.info; + // CHANGE: use "callback" as the callback of the logging function + this.logger[graylogLevel](message, extra, () => callback()); + } + + setConfig(opts) { + this.logger.setConfig(opts.gelfPro); + } +} + +module.exports = exports = GelfTransport; diff --git a/opencti-platform/opencti-graphql/src/config/log-shipping.js b/opencti-platform/opencti-graphql/src/config/log-shipping.js new file mode 100644 index 000000000000..d64bded99690 --- /dev/null +++ b/opencti-platform/opencti-graphql/src/config/log-shipping.js @@ -0,0 +1,50 @@ +import { format } from 'winston'; +import GelfTransport from './gelf-transport'; + +/** + * Create a new log shipping transport. + * @param {Object} conf The transport configuration + * @param {string} conf.logs_shipping_level The minimum log level of messages to send to ship + * @param {string} conf.logs_shipping_env_var_prefix The prefix used to match environment variables. Matching + * variables will be added as meta info to the log data. The value of this property will be stripped from the name + * of the environment variable. + * @param {string} conf.logs_graylog_host The Graylog host to connect to + * @param {number} conf.logs_graylog_port The port to use when connecting to the Graylog host + * @param {'tcp'|'udp'} conf.logs_graylog_adapter The adapter (udp/tcp) to use when connecting to the Graylog host + * @returns {import('winston-gelf')} The newly created log shipping transport + */ +export function createLogShippingTransport(conf) { + return new GelfTransport({ + level: conf.logs_shipping_level, + format: format.combine( + envVarsFormat(conf.logs_shipping_env_var_prefix)(), + format.json(), + ), + gelfPro: { + adapterName: `${conf.logs_graylog_adapter}.js`, // append '.js', as a workaround for https://github.com/evanw/esbuild/issues/3328 + adapterOptions: { + host: conf.logs_graylog_host, + port: conf.logs_graylog_port, + }, + }, + }); +} + +function envVarsFormat(prefix) { + const envVars = findPrefixedEnvVars(prefix); + + return format( + (info) => ({ ...info, ...envVars }) + ); +} + +function findPrefixedEnvVars(prefix) { + return Object.fromEntries( + Object.entries(process.env) + .flatMap(([key, value]) => { + return key.startsWith(prefix) + ? [[key.substring(prefix.length), value]] + : []; + }) + ); +} diff --git a/opencti-platform/opencti-graphql/yarn.lock b/opencti-platform/opencti-graphql/yarn.lock index 0e1f71592543..04b39c1692f8 100644 --- a/opencti-platform/opencti-graphql/yarn.lock +++ b/opencti-platform/opencti-graphql/yarn.lock @@ -8649,6 +8649,15 @@ __metadata: languageName: node linkType: hard +"gelf-pro@npm:1.4.0": + version: 1.4.0 + resolution: "gelf-pro@npm:1.4.0" + dependencies: + lodash: "npm:~4.17.21" + checksum: 10/65fa4a38f1d9c4f48fbbb4bc8cd11fe9333cd26dce387ca96e52ea91b5bfc0eaf5370ab49e7a2db1e7f262d74751a5e5e2c8d5d37fa03aa3b427b1db7b13bcde + languageName: node + linkType: hard + "gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -11469,6 +11478,7 @@ __metadata: fast-glob: "npm:3.3.3" fast-json-patch: "npm:3.1.1" file-type: "npm:19.6.0" + gelf-pro: "npm:1.4.0" github-api: "npm:3.4.0" graphql: "npm:16.9.0" graphql-constraint-directive: "npm:5.4.3" @@ -11530,6 +11540,7 @@ __metadata: vitest: "npm:2.0.5" winston: "npm:3.17.0" winston-daily-rotate-file: "npm:5.0.0" + winston-transport: "npm:4.9.0" ws: "npm:8.18.0" xml2js: "npm:0.6.2" languageName: unknown @@ -14528,6 +14539,17 @@ __metadata: languageName: node linkType: hard +"winston-transport@npm:4.9.0, winston-transport@npm:^4.9.0": + version: 4.9.0 + resolution: "winston-transport@npm:4.9.0" + dependencies: + logform: "npm:^2.7.0" + readable-stream: "npm:^3.6.2" + triple-beam: "npm:^1.3.0" + checksum: 10/5946918720baadd7447823929e94cf0935f92c4cff6d9451c6fcb009bd9d20a3b3df9ad606109e79d1e9f4d2ff678477bf09f81cfefce2025baaf27a617129bb + languageName: node + linkType: hard + "winston-transport@npm:^4.7.0": version: 4.8.0 resolution: "winston-transport@npm:4.8.0" @@ -14539,17 +14561,6 @@ __metadata: languageName: node linkType: hard -"winston-transport@npm:^4.9.0": - version: 4.9.0 - resolution: "winston-transport@npm:4.9.0" - dependencies: - logform: "npm:^2.7.0" - readable-stream: "npm:^3.6.2" - triple-beam: "npm:^1.3.0" - checksum: 10/5946918720baadd7447823929e94cf0935f92c4cff6d9451c6fcb009bd9d20a3b3df9ad606109e79d1e9f4d2ff678477bf09f81cfefce2025baaf27a617129bb - languageName: node - linkType: hard - "winston@npm:3.17.0": version: 3.17.0 resolution: "winston@npm:3.17.0"