-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #252 from serguun42/metrics-and-traces
Add Grafana and Prometheus, update Panel and Backend (Run tests)
- Loading branch information
Showing
52 changed files
with
1,591 additions
and
2,656 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| import rateLimiter from "../util/rate-limiter.js"; | ||
| import logging from "../util/logging.js"; | ||
| import logTooManyRequests from "../util/log-too-many-requests.js"; | ||
| import MongoDispatcher from "../database/dispatcher.js"; | ||
| import readConfig from "../util/read-config.js"; | ||
| import listAllGroups from "./methods/list-groups-all.js"; | ||
| import getCertainGroup from "./methods/get-group.js"; | ||
| import getStartTime from "./methods/get-start-time.js"; | ||
| import getCurrentWeek from "./methods/get-week.js"; | ||
| import getStats from "./methods/get-stats.js"; | ||
|
|
||
| const { DATABASE_NAME } = readConfig(); | ||
|
|
||
| /** @param {import("../types").APIModuleDTO} moduleDTO */ | ||
| export default function coreAPIModule(moduleDTO) { | ||
| const { res, req, path, sendCode, sendPayload } = moduleDTO; | ||
|
|
||
| moduleDTO.mongoDispatcher = new MongoDispatcher(DATABASE_NAME); | ||
|
|
||
| if (rateLimiter(req)) { | ||
| logTooManyRequests(req); | ||
| return sendCode(429); | ||
| } | ||
|
|
||
| res.setHeader("Access-Control-Allow-Origin", "*"); | ||
|
|
||
| if (path[0] !== "api" || path[1] !== "v1.3") return sendPayload(400, { error: true, message: "No such API version" }); | ||
|
|
||
| switch (path[2]) { | ||
| case "groups": | ||
| switch (path[3]) { | ||
| case "all": | ||
| listAllGroups(moduleDTO); | ||
| break; | ||
|
|
||
| case "certain": | ||
| getCertainGroup(moduleDTO); | ||
| break; | ||
|
|
||
| default: | ||
| sendPayload(404, { error: true, message: "No such method" }); | ||
| break; | ||
| } | ||
| break; | ||
|
|
||
| case "time": | ||
| switch (path[3]) { | ||
| case "startTime": | ||
| getStartTime(moduleDTO); | ||
| break; | ||
|
|
||
| case "week": | ||
| getCurrentWeek(moduleDTO); | ||
| break; | ||
|
|
||
| default: | ||
| sendPayload(404, { error: true, message: "No such method" }); | ||
| break; | ||
| } | ||
| break; | ||
|
|
||
| case "stats": | ||
| getStats(moduleDTO); | ||
| break; | ||
|
|
||
| case "ping": | ||
| sendPayload(200, { | ||
| message: "pong" | ||
| }); | ||
| break; | ||
|
|
||
| case "logs": | ||
| switch (path[3]) { | ||
| case "post": | ||
| sendCode(201); | ||
| break; | ||
|
|
||
| default: | ||
| sendPayload(404, { error: true, message: "No such method" }); | ||
| break; | ||
| } | ||
| break; | ||
|
|
||
| default: | ||
| sendPayload(404, { error: true, message: "No such method" }); | ||
| break; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| import logging from "../../util/logging.js"; | ||
|
|
||
| /** @param {import("../../types").APIModuleDTO} moduleDTO */ | ||
| export default function getCertainGroup({ mongoDispatcher, sendCode, sendPayload, queries }) { | ||
| if (!mongoDispatcher) return sendCode(500); | ||
|
|
||
| if (typeof queries["name"] !== "string") | ||
| return sendPayload(400, { error: true, message: "No required <name> parameter" }); | ||
|
|
||
| const selector = { | ||
| groupName: queries["name"] | ||
| }; | ||
|
|
||
| if (queries["suffix"] && typeof queries["suffix"] === "string") selector["groupSuffix"] = queries["suffix"]; | ||
|
|
||
| mongoDispatcher | ||
| .callDB() | ||
| .then((DB) => | ||
| DB.collection("study_groups").find(selector).sort({ groupName: 1, groupSuffix: 1 }).project({ _id: 0 }).toArray() | ||
| ) | ||
| .then((groups) => { | ||
| if (!groups.length) sendPayload(404, []); | ||
| else sendPayload(200, groups); | ||
| }) | ||
| .catch((e) => { | ||
| logging("Error getting certain group", e); | ||
| sendCode(500); | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| import logging from "../../util/logging.js"; | ||
|
|
||
| /** @param {import("../../types").APIModuleDTO} moduleDTO */ | ||
| export default function getStartTime({ mongoDispatcher, sendCode, sendPayload }) { | ||
| if (!mongoDispatcher) return sendCode(500); | ||
|
|
||
| mongoDispatcher | ||
| .callDB() | ||
| .then((DB) => DB.collection("params").findOne({ name: "start_of_weeks" })) | ||
| .then((found) => { | ||
| if (found && found.value) sendPayload(200, new Date(found.value).toISOString()); | ||
| else sendPayload(404, "Property start_of_weeks not found"); | ||
| }) | ||
| .catch((e) => { | ||
| logging("Error listing groups", e); | ||
| sendCode(500); | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| import logging from "../../util/logging.js"; | ||
|
|
||
| /** @param {import("../../types").APIModuleDTO} moduleDTO */ | ||
| export default function getStats({ mongoDispatcher, sendCode, sendPayload }) { | ||
| if (!mongoDispatcher) return sendCode(500); | ||
|
|
||
| mongoDispatcher | ||
| .callDB() | ||
| .then((DB) => | ||
| DB.collection("params") | ||
| .findOne({ name: "scrapper_updated_date" }) | ||
| .then((scrapperUpdatedDate) => { | ||
| if (scrapperUpdatedDate) | ||
| return DB.collection("study_groups") | ||
| .countDocuments() | ||
| .then((groupsCount) => { | ||
| sendPayload(200, { | ||
| scrapperUpdatedDate: new Date(scrapperUpdatedDate.value || 0).toISOString(), | ||
| groupsCount | ||
| }); | ||
| }); | ||
| else sendPayload(404, "Property scrapper_updated_date not found"); | ||
| }) | ||
| ) | ||
| .catch((e) => { | ||
| logging("Error listing groups", e); | ||
| sendCode(500); | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| import logging from "../../util/logging.js"; | ||
|
|
||
| const SECOND = 1000; | ||
| const MINUTE = SECOND * 60; | ||
| const HOUR = MINUTE * 60; | ||
|
|
||
| /** @param {import("../../types").APIModuleDTO} moduleDTO */ | ||
| export default function getCurrentWeek({ mongoDispatcher, sendCode, sendPayload }) { | ||
| if (!mongoDispatcher) return sendCode(500); | ||
|
|
||
| mongoDispatcher | ||
| .callDB() | ||
| .then((DB) => DB.collection("params").findOne({ name: "start_of_weeks" })) | ||
| .then((found) => { | ||
| if (found && found.value) sendPayload(200, Math.ceil((Date.now() - found.value) / (7 * 24 * HOUR))); | ||
| else sendPayload(404, "Cannot compute current week"); | ||
| }) | ||
| .catch((e) => { | ||
| logging("Error listing groups", e); | ||
| sendCode(500); | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| import logging from "../../util/logging.js"; | ||
|
|
||
| /** @param {import("../../types").APIModuleDTO} moduleDTO */ | ||
| export default function listAllGroups({ mongoDispatcher, sendCode, sendPayload }) { | ||
| if (!mongoDispatcher) return sendCode(500); | ||
|
|
||
| mongoDispatcher | ||
| .callDB() | ||
| .then((DB) => | ||
| DB.collection("study_groups") | ||
| .find() | ||
| .sort({ groupName: 1, groupSuffix: 1 }) | ||
| .project({ groupName: 1, groupSuffix: 1, _id: 0 }) | ||
| .toArray() | ||
| ) | ||
| .then((names) => sendPayload(200, names)) | ||
| .catch((e) => { | ||
| logging("Error listing groups", e); | ||
| sendCode(500); | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,80 +1,95 @@ | ||
| const http = require("http"); | ||
| import { createServer, STATUS_CODES } from "node:http"; | ||
| import promClient from "prom-client"; | ||
| import { parsePath, parseQuery } from "./util/urls-and-cookies.js"; | ||
| import coreAPIModule from "./api/core.js"; | ||
| import logging from "./util/logging.js"; | ||
|
|
||
| /** | ||
| * @param {{[code: string]: string}} iStatusCodes | ||
| * @returns {{[code: number]: string}} | ||
| */ | ||
| const GetStatusCodes = (iStatusCodes) => { | ||
| const newCodes = {}; | ||
| const register = new promClient.Registry(); | ||
| register.setDefaultLabels({ | ||
| app: "backend-server" | ||
| }); | ||
|
|
||
| Object.keys(iStatusCodes).forEach((code) => (newCodes[code] = `${code} ${iStatusCodes[code]}`)); | ||
| promClient.collectDefaultMetrics({ register }); | ||
|
|
||
| return newCodes; | ||
| }; | ||
| const httpRequestDurationMicroseconds = new promClient.Histogram({ | ||
| name: "http_request_duration_seconds", | ||
| help: "Duration of HTTP requests in microseconds", | ||
| labelNames: ["method", "route", "code"], | ||
| buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10] | ||
| }); | ||
|
|
||
| /** | ||
| * HTTP Response Statuses | ||
| * @type {{[code: number]: string}} | ||
| */ | ||
| const STATUSES = GetStatusCodes(http.STATUS_CODES); | ||
| register.registerMetric(httpRequestDurationMicroseconds); | ||
|
|
||
| const UTIL = require("./utils/urls-and-cookies"); | ||
| export default function createBackendServer() { | ||
| return createServer((req, res) => { | ||
| const markMetrics = httpRequestDurationMicroseconds.startTimer(); | ||
|
|
||
| const CreateServer = () => | ||
| http.createServer((req, res) => { | ||
| const path = UTIL.ParsePath(req.url); | ||
| const queries = UTIL.ParseQuery(UTIL.SafeURL(req.url).search); | ||
| const cookies = UTIL.ParseCookie(req.headers); | ||
| const path = parsePath(req.url); | ||
| const queries = parseQuery(req.url); | ||
|
|
||
| res.setHeader("Content-Type", "charset=UTF-8"); | ||
| res.setHeader("Content-Type", "text/plain; charset=UTF-8"); | ||
|
|
||
| /** | ||
| * @param {number} iCode | ||
| * @param {string | Buffer | ReadStream | Object} iData | ||
| * @returns {false} | ||
| * @param {number} code | ||
| * @param {string | Buffer | ReadStream | Object} data | ||
| */ | ||
| const GlobalSendCustom = (iCode, iData) => { | ||
| res.statusCode = iCode; | ||
| const sendPayload = (code, data) => { | ||
| res.statusCode = code; | ||
|
|
||
| if (iData instanceof Buffer || typeof iData == "string") { | ||
| const dataToSend = iData.toString(); | ||
| if (data instanceof Buffer || typeof data == "string") { | ||
| const dataToSend = data.toString(); | ||
|
|
||
| res.end(dataToSend); | ||
| } else { | ||
| const dataToSend = JSON.stringify(iData); | ||
| res.setHeader("Content-Type", UTIL.SetCompleteMIMEType(".json")); | ||
| const dataToSend = JSON.stringify(data); | ||
| res.setHeader("Content-Type", "application/json; charset=UTF-8"); | ||
|
|
||
| res.end(dataToSend); | ||
| } | ||
|
|
||
| return false; | ||
| markMetrics({ | ||
| method: req.method, | ||
| route: `/${path.join("/")}`, | ||
| code | ||
| }); | ||
| }; | ||
|
|
||
| /** | ||
| * @param {number} iCode | ||
| * @returns {false} | ||
| */ | ||
| const GlobalSend = (iCode) => { | ||
| res.statusCode = iCode || 200; | ||
| res.end(STATUSES[iCode || 500]); | ||
| return false; | ||
| /** @param {number} code */ | ||
| const sendCode = (code) => { | ||
| res.statusCode = code || 200; | ||
| res.end(`${code || 500} ${STATUS_CODES[code || 500]}`); | ||
|
|
||
| markMetrics({ | ||
| method: req.method, | ||
| route: path, | ||
| code | ||
| }); | ||
| }; | ||
|
|
||
| /** @type {import("./types").ModuleCallingObjectType} */ | ||
| const CALLING_PROPS = { | ||
| if (path[0] === "metrics") { | ||
| register | ||
| .metrics() | ||
| .then((metrics) => { | ||
| res.setHeader("Content-Type", register.contentType); | ||
| res.end(metrics); | ||
| }) | ||
| .catch((e) => { | ||
| logging(e); | ||
| sendCode(500); | ||
| }); | ||
| return; | ||
| } | ||
|
|
||
| if (path[0] !== "api") return sendCode(404); | ||
| coreAPIModule({ | ||
| req, | ||
| res, | ||
| path, | ||
| queries, | ||
| cookies, | ||
| GlobalSend, | ||
| GlobalSendCustom | ||
| }; | ||
|
|
||
| if (path[0] === "api") return require("./pages/api")(CALLING_PROPS); | ||
| else return GlobalSend(404); | ||
| sendCode, | ||
| sendPayload | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| if (process.env.NODE_ENV !== "test") CreateServer().listen(80); | ||
|
|
||
| module.exports = CreateServer; | ||
| if (process.env.NODE_ENV !== "test") createBackendServer().listen(80); |
Oops, something went wrong.