From a3d45e46e662e60b14e30605e1c7266dab989902 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Sun, 19 Jan 2025 22:05:28 +0000 Subject: [PATCH] Set references to frictionlesss tokens. Breakdown scores into components. Add to score when user access policy set. --- .../src/controllers/auth.ts | 7 ++- .../src/apiExpressRouterFactory.ts | 20 +++++- .../apiExpressDefaultEndpointAdapter.ts | 16 ++++- .../apiExpressEndpointAdapter.ts | 3 +- .../src}/errorHandler.ts | 0 packages/api-express-router/src/index.ts | 2 + .../src/tests/unit}/errorHandler.unit.test.ts | 2 +- packages/database/src/databases/provider.ts | 24 ++++--- packages/provider/package.json | 1 + packages/provider/src/api/captcha.ts | 57 ++++++++++++----- packages/provider/src/api/domainMiddleware.ts | 2 +- .../provider/src/api/headerCheckMiddleware.ts | 3 - packages/provider/src/api/public.ts | 4 +- packages/provider/src/api/verify.ts | 3 +- packages/provider/src/index.ts | 1 - .../frictionless/frictionlessPenalties.ts | 15 +++++ .../frictionless/frictionlessTasksUtils.ts | 27 ++++++++ .../src/tasks/imgCaptcha/imgCaptchaTasks.ts | 29 +++++++-- .../provider/src/tasks/powCaptcha/powTasks.ts | 27 ++++++-- .../imgCaptcha/imgCaptchaTasks.unit.test.ts | 2 +- packages/provider/tsconfig.cjs.json | 9 ++- packages/provider/tsconfig.json | 9 ++- packages/types-database/src/index.ts | 1 + .../src/provider/pendingCaptchaRequest.ts | 27 ++++++++ packages/types-database/src/types/provider.ts | 63 ++++++++++++++----- packages/types/src/provider/api.ts | 15 +---- .../apiInsertManyRulesArgsSchema.ts | 6 +- .../insertMany/apiInsertManyRulesEndpoint.ts | 27 +++++++- .../rules/mongoose/rulesMongooseStorage.ts | 19 +++++- 29 files changed, 329 insertions(+), 92 deletions(-) rename packages/{provider/src/api => api-express-router/src}/errorHandler.ts (100%) rename packages/{provider/src/tests/unit/api => api-express-router/src/tests/unit}/errorHandler.unit.test.ts (98%) create mode 100644 packages/provider/src/tasks/frictionless/frictionlessPenalties.ts create mode 100644 packages/provider/src/tasks/frictionless/frictionlessTasksUtils.ts create mode 100644 packages/types-database/src/provider/pendingCaptchaRequest.ts diff --git a/demos/client-example-server/src/controllers/auth.ts b/demos/client-example-server/src/controllers/auth.ts index 24c4c115b6..3dfaf73dd1 100644 --- a/demos/client-example-server/src/controllers/auth.ts +++ b/demos/client-example-server/src/controllers/auth.ts @@ -61,12 +61,13 @@ const verify = async ( }), }); - const verified = (await response.json()).verified; - console.log("Verified", verified); - return verified; + const verifiedResponse = await response.json(); + console.log(verifiedResponse); + return verifiedResponse.verified; } // verify using the TypeScript library const verified = await prosopoServer.isVerified(token); + console.log(verified); return verified; }; diff --git a/packages/api-express-router/src/apiExpressRouterFactory.ts b/packages/api-express-router/src/apiExpressRouterFactory.ts index ae4a999b6a..66b3c4696d 100644 --- a/packages/api-express-router/src/apiExpressRouterFactory.ts +++ b/packages/api-express-router/src/apiExpressRouterFactory.ts @@ -13,8 +13,14 @@ // limitations under the License. import type { ApiRoute, ApiRoutesProvider } from "@prosopo/api-route"; -import { type Request, type Response, Router } from "express"; +import { + type NextFunction, + type Request, + type Response, + Router, +} from "express"; import type { ApiExpressEndpointAdapter } from "./endpointAdapter/apiExpressEndpointAdapter.js"; +import { handleErrors } from "./errorHandler.js"; class ApiExpressRouterFactory { public createRouter( @@ -26,6 +32,11 @@ class ApiExpressRouterFactory { this.registerRoutes(router, apiRoutes, apiEndpointAdapter); + // Your error handler should always be at the end of your application stack. Apparently it means not only after all + // app.use() but also after all your app.get() and app.post() calls. + // https://stackoverflow.com/a/62358794/1178971 + router.use(handleErrors); + return router; } @@ -46,11 +57,16 @@ class ApiExpressRouterFactory { ): void { router.post( route.path, - async (request: Request, response: Response): Promise => { + async ( + request: Request, + response: Response, + next: NextFunction, + ): Promise => { await apiEndpointAdapter.handleRequest( route.endpoint, request, response, + next, ); }, ); diff --git a/packages/api-express-router/src/endpointAdapter/apiExpressDefaultEndpointAdapter.ts b/packages/api-express-router/src/endpointAdapter/apiExpressDefaultEndpointAdapter.ts index b7988ad5da..3ca23f9419 100644 --- a/packages/api-express-router/src/endpointAdapter/apiExpressDefaultEndpointAdapter.ts +++ b/packages/api-express-router/src/endpointAdapter/apiExpressDefaultEndpointAdapter.ts @@ -13,8 +13,8 @@ // limitations under the License. import type { ApiEndpoint } from "@prosopo/api-route"; -import type { Logger } from "@prosopo/common"; -import type { Request, Response } from "express"; +import { type Logger, ProsopoApiError } from "@prosopo/common"; +import type { NextFunction, Request, Response } from "express"; import type { ZodType } from "zod"; import type { ApiExpressEndpointAdapter } from "./apiExpressEndpointAdapter.js"; @@ -28,10 +28,20 @@ class ApiExpressDefaultEndpointAdapter implements ApiExpressEndpointAdapter { endpoint: ApiEndpoint, request: Request, response: Response, + next: NextFunction, ): Promise { + let args: unknown; try { - const args = endpoint.getRequestArgsSchema()?.parse(request.body); + args = endpoint.getRequestArgsSchema()?.parse(request.body); + } catch (error) { + return next( + new ProsopoApiError("API.PARSE_ERROR", { + context: { code: 400, error: error }, + }), + ); + } + try { const apiEndpointResponse = await endpoint.processRequest(args); response.json(apiEndpointResponse); diff --git a/packages/api-express-router/src/endpointAdapter/apiExpressEndpointAdapter.ts b/packages/api-express-router/src/endpointAdapter/apiExpressEndpointAdapter.ts index 6c326c13a4..66e7f671a3 100644 --- a/packages/api-express-router/src/endpointAdapter/apiExpressEndpointAdapter.ts +++ b/packages/api-express-router/src/endpointAdapter/apiExpressEndpointAdapter.ts @@ -13,7 +13,7 @@ // limitations under the License. import type { ApiEndpoint } from "@prosopo/api-route"; -import type { Request, Response } from "express"; +import type { NextFunction, Request, Response } from "express"; import type { ZodType } from "zod"; interface ApiExpressEndpointAdapter { @@ -21,6 +21,7 @@ interface ApiExpressEndpointAdapter { endpoint: ApiEndpoint, request: Request, response: Response, + next: NextFunction, ): Promise; } diff --git a/packages/provider/src/api/errorHandler.ts b/packages/api-express-router/src/errorHandler.ts similarity index 100% rename from packages/provider/src/api/errorHandler.ts rename to packages/api-express-router/src/errorHandler.ts diff --git a/packages/api-express-router/src/index.ts b/packages/api-express-router/src/index.ts index b516c3c934..120bd3f332 100644 --- a/packages/api-express-router/src/index.ts +++ b/packages/api-express-router/src/index.ts @@ -31,3 +31,5 @@ export { createApiExpressDefaultEndpointAdapter, type ApiExpressEndpointAdapter, }; + +export * from "./errorHandler.js"; diff --git a/packages/provider/src/tests/unit/api/errorHandler.unit.test.ts b/packages/api-express-router/src/tests/unit/errorHandler.unit.test.ts similarity index 98% rename from packages/provider/src/tests/unit/api/errorHandler.unit.test.ts rename to packages/api-express-router/src/tests/unit/errorHandler.unit.test.ts index 9d64596c3a..11f78fe224 100644 --- a/packages/provider/src/tests/unit/api/errorHandler.unit.test.ts +++ b/packages/api-express-router/src/tests/unit/errorHandler.unit.test.ts @@ -19,7 +19,7 @@ import type { NextFunction, Request, Response } from "express"; // limitations under the License. import { describe, expect, it, vi } from "vitest"; import { ZodError } from "zod"; -import { handleErrors } from "../../../api/errorHandler.js"; +import { handleErrors } from "../../errorHandler.js"; describe("handleErrors", () => { it("should handle ProsopoApiError", () => { diff --git a/packages/database/src/databases/provider.ts b/packages/database/src/databases/provider.ts index c0cfc5b356..49df8a42f4 100644 --- a/packages/database/src/databases/provider.ts +++ b/packages/database/src/databases/provider.ts @@ -27,7 +27,6 @@ import { type DatasetWithIdsAndTree, DatasetWithIdsAndTreeSchema, type Hash, - type PendingCaptchaRequest, type PoWChallengeComponents, type PoWChallengeId, type RequestHeaders, @@ -41,11 +40,13 @@ import { ClientRecordSchema, DatasetRecordSchema, type FrictionlessToken, + type FrictionlessTokenId, FrictionlessTokenRecordSchema, type IPBlockRuleRecord, IPBlockRuleRecordSchema, type IProviderDatabase, type IUserDataSlim, + type PendingCaptchaRequest, type PendingCaptchaRequestMongoose, PendingRecordSchema, type PoWCaptchaRecord, @@ -566,7 +567,7 @@ export class ProviderDatabase * @param providerSignature * @param ipAddress * @param headers - * @param score + * @param frictionlessTokenId * @param serverChecked * @param userSubmitted * @param storedStatus @@ -580,7 +581,7 @@ export class ProviderDatabase providerSignature: string, ipAddress: bigint, headers: RequestHeaders, - score?: number, + frictionlessTokenId?: FrictionlessTokenId, serverChecked = false, userSubmitted = false, storedStatus: StoredStatus = StoredStatusNames.notStored, @@ -600,7 +601,7 @@ export class ProviderDatabase providerSignature, userSignature, lastUpdatedTimestamp: Date.now(), - score, + frictionlessTokenId, }; try { @@ -941,9 +942,18 @@ export class ProviderDatabase return doc._id; } + /** Update a frictionless token record */ + async updateFrictionlessTokenRecord( + tokenId: FrictionlessTokenId, + updates: Partial, + ): Promise { + const filter: Pick = { _id: tokenId }; + await this.tables.frictionlessToken.updateOne(filter, updates); + } + /** Get a frictionless token record */ async getFrictionlessTokenRecordByTokenId( - tokenId: ObjectId, + tokenId: FrictionlessTokenId, ): Promise { const filter: Pick = { _id: tokenId }; const doc = @@ -1014,7 +1024,7 @@ export class ProviderDatabase requestedAtTimestamp: number, ipAddress: bigint, headers: RequestHeaders, - score?: number, + frictionlessTokenId?: FrictionlessTokenId, ): Promise { if (!isHex(requestHash)) { throw new ProsopoDBError("DATABASE.INVALID_HASH", { @@ -1033,7 +1043,7 @@ export class ProviderDatabase requestedAtTimestamp: new Date(requestedAtTimestamp), headers, ipAddress, - score, + frictionlessTokenId, }; await this.tables?.pending.updateOne( { requestHash: requestHash }, diff --git a/packages/provider/package.json b/packages/provider/package.json index 3ff0a20e0e..c0664681d3 100644 --- a/packages/provider/package.json +++ b/packages/provider/package.json @@ -28,6 +28,7 @@ "@polkadot/util": "12.6.2", "@polkadot/util-crypto": "12.6.2", "@prosopo/api-route": "2.3.1", + "@prosopo/api-express-router": "2.3.1", "@prosopo/common": "2.3.1", "@prosopo/config": "2.3.1", "@prosopo/keyring": "2.3.1", diff --git a/packages/provider/src/api/captcha.ts b/packages/provider/src/api/captcha.ts index 9d35c39e88..2717a514b4 100644 --- a/packages/provider/src/api/captcha.ts +++ b/packages/provider/src/api/captcha.ts @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. import { validateAddress } from "@polkadot/util-crypto/address"; +import { handleErrors } from "@prosopo/api-express-router"; import { ProsopoApiError } from "@prosopo/common"; import { parseCaptchaAssets } from "@prosopo/datasets"; import { @@ -34,15 +35,19 @@ import { SubmitPowCaptchaSolutionBody, type SubmitPowCaptchaSolutionBodyTypeOutput, } from "@prosopo/types"; +import type { FrictionlessTokenId } from "@prosopo/types-database"; import type { ProviderEnvironment } from "@prosopo/types-env"; import { createImageCaptchaConfigResolver } from "@prosopo/user-access-policy"; import { flatten } from "@prosopo/util"; import express, { type Router } from "express"; import type { ObjectId } from "mongoose"; import { getBotScore } from "../tasks/detection/getBotScore.js"; +import { + PENALTY_ACCESS_RULE, + PENALTY_OLD_TIMESTAMP, +} from "../tasks/frictionless/frictionlessPenalties.js"; import { Tasks } from "../tasks/tasks.js"; import { getIPAddress } from "../util.js"; -import { handleErrors } from "./errorHandler.js"; const DEFAULT_FRICTIONLESS_THRESHOLD = 0.5; const TEN_MINUTES = 60 * 10 * 1000; @@ -125,7 +130,7 @@ export function prosopoRouter(env: ProviderEnvironment): Router { ); const clientSettings = await tasks.db.getClientRecord(dapp); - let score: number | undefined; + let frictionlessTokenId: FrictionlessTokenId | undefined; if (sessionId) { const sessionRecord = await tasks.db.checkAndRemoveSession(sessionId); @@ -134,10 +139,9 @@ export function prosopoRouter(env: ProviderEnvironment): Router { await tasks.db.getFrictionlessTokenRecordByTokenId( sessionRecord.tokenId, ); - // there should be a token record. Remove any lScore before storing downstream - score = tokenRecord - ? tokenRecord.score - (tokenRecord.lScore || 0) - : 1; + frictionlessTokenId = tokenRecord + ? (tokenRecord._id as FrictionlessTokenId) + : undefined; } } else if (!(clientSettings?.settings?.captchaType === "image")) { // Throw an error if an image captcha has been requested without a session and the client is not configured for image captchas @@ -155,7 +159,7 @@ export function prosopoRouter(env: ProviderEnvironment): Router { ipAddress, flatten(req.headers), captchaConfig, - score, + frictionlessTokenId, ); const captchaResponse: CaptchaResponseBody = { [ApiParams.status]: "ok", @@ -308,7 +312,7 @@ export function prosopoRouter(env: ProviderEnvironment): Router { ); } - let score: number | undefined; + let frictionlessTokenId: FrictionlessTokenId | undefined; if (sessionId) { const sessionRecord = await tasks.db.checkAndRemoveSession(sessionId); @@ -330,8 +334,9 @@ export function prosopoRouter(env: ProviderEnvironment): Router { const tokenRecord = await tasks.db.getFrictionlessTokenRecordByTokenId( sessionRecord.tokenId, ); - // there should be a token record. Remove any lScore before storing downstream - score = tokenRecord ? tokenRecord.score - (tokenRecord.lScore || 0) : 1; + frictionlessTokenId = tokenRecord + ? (tokenRecord._id as FrictionlessTokenId) + : undefined; } else if (!(clientSettings?.settings?.captchaType === "pow")) { // Throw an error if a pow captcha has been requested without a session and the client is not configured for pow captchas return next( @@ -374,7 +379,7 @@ export function prosopoRouter(env: ProviderEnvironment): Router { challenge.providerSignature, getIPAddress(req.ip || "").bigInt(), flatten(req.headers), - score, + frictionlessTokenId, ); const getPowCaptchaResponse: GetPowCaptchaResponse = { @@ -449,8 +454,6 @@ export function prosopoRouter(env: ProviderEnvironment): Router { try { validateAddress(user, false, 42); - validateAddress(dapp, false, 42); - const clientRecord = await tasks.db.getClientRecord(dapp); if (!clientRecord) { @@ -515,7 +518,7 @@ export function prosopoRouter(env: ProviderEnvironment): Router { const { baseBotScore, timestamp } = await getBotScore(token); - const botScore = baseBotScore + lScore; + let botScore = baseBotScore + lScore; const clientConfig = await tasks.db.getClientRecord(dapp); const botThreshold = @@ -527,7 +530,10 @@ export function prosopoRouter(env: ProviderEnvironment): Router { token, score: botScore, threshold: botThreshold, - ...(lScore && { lScore }), + scoreComponents: { + baseScore: baseBotScore, + ...(lScore && { lScore }), + }, }); // If the timestamp is older than 10 minutes, send an image captcha @@ -536,6 +542,14 @@ export function prosopoRouter(env: ProviderEnvironment): Router { "Timestamp is older than 10 minutes", new Date(timestamp), ); + botScore += PENALTY_OLD_TIMESTAMP; + await tasks.db.updateFrictionlessTokenRecord(tokenId, { + score: botScore, + scoreComponents: { + baseScore: baseBotScore, + timeout: PENALTY_OLD_TIMESTAMP, + }, + }); return res.json( await tasks.frictionlessManager.sendImageCaptcha(tokenId), ); @@ -544,6 +558,7 @@ export function prosopoRouter(env: ProviderEnvironment): Router { // Check if the IP address is blocked const ipAddress = getIPAddress(req.ip || ""); + // If the user or IP address has an image captcha config defined, send an image captcha const imageCaptchaConfigDefined = await imageCaptchaConfigResolver.isConfigDefined( dapp, @@ -551,8 +566,17 @@ export function prosopoRouter(env: ProviderEnvironment): Router { user, ); - if (imageCaptchaConfigDefined) + if (imageCaptchaConfigDefined) { + botScore += PENALTY_ACCESS_RULE; + await tasks.db.updateFrictionlessTokenRecord(tokenId, { + score: botScore, + scoreComponents: { + baseScore: baseBotScore, + accessPolicy: PENALTY_ACCESS_RULE, + }, + }); return res.json(tasks.frictionlessManager.sendImageCaptcha(tokenId)); + } // If the bot score is greater than the threshold, send an image captcha if (Number(botScore) > botThreshold) { @@ -561,6 +585,7 @@ export function prosopoRouter(env: ProviderEnvironment): Router { ); } + // Otherwise, send a PoW captcha const response = await tasks.frictionlessManager.sendPowCaptcha(tokenId); diff --git a/packages/provider/src/api/domainMiddleware.ts b/packages/provider/src/api/domainMiddleware.ts index 55e6c376d9..e001a4d1bd 100644 --- a/packages/provider/src/api/domainMiddleware.ts +++ b/packages/provider/src/api/domainMiddleware.ts @@ -13,12 +13,12 @@ // limitations under the License. import { validateAddress } from "@polkadot/util-crypto"; +import { handleErrors } from "@prosopo/api-express-router"; import { ProsopoApiError } from "@prosopo/common"; import type { ProviderEnvironment } from "@prosopo/types-env"; import type { NextFunction, Request, Response } from "express"; import { ZodError } from "zod"; import { Tasks } from "../tasks/index.js"; -import { handleErrors } from "./errorHandler.js"; export const domainMiddleware = (env: ProviderEnvironment) => { const tasks = new Tasks(env); diff --git a/packages/provider/src/api/headerCheckMiddleware.ts b/packages/provider/src/api/headerCheckMiddleware.ts index 68826fd419..130dfbd6ff 100644 --- a/packages/provider/src/api/headerCheckMiddleware.ts +++ b/packages/provider/src/api/headerCheckMiddleware.ts @@ -11,11 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import { ProsopoApiError } from "@prosopo/common"; import type { ProviderEnvironment } from "@prosopo/types-env"; import type { NextFunction, Request, Response } from "express"; -import { ZodError } from "zod"; -import { handleErrors } from "./errorHandler.js"; export const headerCheckMiddleware = (env: ProviderEnvironment) => { return async (req: Request, res: Response, next: NextFunction) => { diff --git a/packages/provider/src/api/public.ts b/packages/provider/src/api/public.ts index 543a602d78..bf6aafca34 100644 --- a/packages/provider/src/api/public.ts +++ b/packages/provider/src/api/public.ts @@ -1,3 +1,4 @@ +import { handleErrors } from "@prosopo/api-express-router"; // Copyright 2021-2024 Prosopo (UK) Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,9 +18,6 @@ import type { ProviderEnvironment } from "@prosopo/types-env"; import { version } from "@prosopo/util"; import express, { type Router } from "express"; import { Tasks } from "../tasks/tasks.js"; -import { handleErrors } from "./errorHandler.js"; - -const DEFAULT_FRICTIONLESS_THRESHOLD = 0.5; /** * Returns a router connected to the database which can interact with the Proposo protocol diff --git a/packages/provider/src/api/verify.ts b/packages/provider/src/api/verify.ts index d917ac41e4..199e5788b7 100644 --- a/packages/provider/src/api/verify.ts +++ b/packages/provider/src/api/verify.ts @@ -13,6 +13,7 @@ // limitations under the License. import { validateAddress } from "@polkadot/util-crypto/address"; +import { handleErrors } from "@prosopo/api-express-router"; import { ProsopoApiError } from "@prosopo/common"; import { ApiParams, @@ -29,7 +30,6 @@ import type { ProviderEnvironment } from "@prosopo/types-env"; import express, { type Router } from "express"; import { Tasks } from "../tasks/tasks.js"; import { verifySignature } from "./authMiddleware.js"; -import { handleErrors } from "./errorHandler.js"; /** * Returns a router connected to the database which can interact with the Proposo protocol @@ -101,6 +101,7 @@ export function prosopoVerifyRouter(env: ProviderEnvironment): Router { parsed.maxVerifiedTime, ); + tasks.logger.debug(response); const verificationResponse: ImageVerificationResponse = { [ApiParams.status]: req.t(response.status), [ApiParams.verified]: response[ApiParams.verified], diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index 69c42ad3a9..456d323119 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -16,7 +16,6 @@ export * from "./util.js"; export * from "./api/block.js"; export * from "./api/captcha.js"; export * from "./api/verify.js"; -export * from "./api/errorHandler.js"; export * from "./api/authMiddleware.js"; export * from "./api/public.js"; export * from "./api/domainMiddleware.js"; diff --git a/packages/provider/src/tasks/frictionless/frictionlessPenalties.ts b/packages/provider/src/tasks/frictionless/frictionlessPenalties.ts new file mode 100644 index 0000000000..169dea0045 --- /dev/null +++ b/packages/provider/src/tasks/frictionless/frictionlessPenalties.ts @@ -0,0 +1,15 @@ +// Copyright 2021-2024 Prosopo (UK) Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +export const PENALTY_OLD_TIMESTAMP = 0.2; +export const PENALTY_ACCESS_RULE = 0.5; diff --git a/packages/provider/src/tasks/frictionless/frictionlessTasksUtils.ts b/packages/provider/src/tasks/frictionless/frictionlessTasksUtils.ts new file mode 100644 index 0000000000..fc58b3a48c --- /dev/null +++ b/packages/provider/src/tasks/frictionless/frictionlessTasksUtils.ts @@ -0,0 +1,27 @@ +// Copyright 2021-2024 Prosopo (UK) Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +import type { ScoreComponents } from "@prosopo/types-database"; + +export const computeFrictionlessScore = ( + scoreComponents: ScoreComponents, +): number => { + return Number( + Math.min( + 1, + Object.values(scoreComponents) + .filter((x) => x !== undefined) + .reduce((acc, val) => acc + val, 0), + ).toFixed(2), + ); +}; diff --git a/packages/provider/src/tasks/imgCaptcha/imgCaptchaTasks.ts b/packages/provider/src/tasks/imgCaptcha/imgCaptchaTasks.ts index 773160faed..515d0e7cfe 100644 --- a/packages/provider/src/tasks/imgCaptcha/imgCaptchaTasks.ts +++ b/packages/provider/src/tasks/imgCaptcha/imgCaptchaTasks.ts @@ -30,18 +30,20 @@ import { type Hash, type IPAddress, type ImageVerificationResponse, - type PendingCaptchaRequest, type ProsopoCaptchaCountConfigSchemaOutput, type ProsopoConfigOutput, type RequestHeaders, } from "@prosopo/types"; import type { + FrictionlessTokenId, IProviderDatabase, + PendingCaptchaRequest, UserCommitment, } from "@prosopo/types-database"; import { at } from "@prosopo/util"; import { checkLangRules } from "../../rules/lang.js"; import { shuffleArray } from "../../util.js"; +import { computeFrictionlessScore } from "../frictionless/frictionlessTasksUtils.js"; import { buildTreeAndGetCommitmentId } from "./imgCaptchaTasksUtils.js"; export class ImgCaptchaManager { @@ -88,7 +90,7 @@ export class ImgCaptchaManager { ipAddress: IPAddress, headers: RequestHeaders, captchaConfig: ProsopoCaptchaCountConfigSchemaOutput, - score?: number, + frictionlessTokenId?: FrictionlessTokenId, ): Promise<{ captchas: Captcha[]; requestHash: string; @@ -156,7 +158,7 @@ export class ImgCaptchaManager { currentTime, ipAddress.bigInt(), headers, - score, + frictionlessTokenId, ); return { captchas, @@ -172,7 +174,7 @@ export class ImgCaptchaManager { * @param {string} dappAccount * @param {string} requestHash * @param {JSON} captchas - * @param {string} userRequestHashSignature + * @param userTimestampSignature * @param timestamp * @param providerRequestHashSignature * @param ipAddress @@ -268,7 +270,7 @@ export class ImgCaptchaManager { requestedAtTimestamp: timestamp, ipAddress, headers, - score: pendingRecord.score, + frictionlessTokenId: pendingRecord.frictionlessTokenId, }; await this.db.storeUserImageCaptchaSolution(receivedCaptchas, commit); @@ -454,16 +456,31 @@ export class ImgCaptchaManager { return { status: "API.USER_NOT_VERIFIED_TIME_EXPIRED", verified: false, - score: solution.score, }; } } const isApproved = solution.result.status === CaptchaStatus.approved; + + let score: number | undefined; + if (solution.frictionlessTokenId) { + const tokenRecord = await this.db.getFrictionlessTokenRecordByTokenId( + solution.frictionlessTokenId, + ); + if (tokenRecord) { + score = computeFrictionlessScore(tokenRecord?.scoreComponents); + this.logger.info({ + tscoreComponents: tokenRecord?.scoreComponents, + score: score, + }); + } + } + return { status: isApproved ? "API.USER_VERIFIED" : "API.USER_NOT_VERIFIED", verified: isApproved, commitmentId: solution.id.toString(), + ...(score ? { score } : {}), }; } diff --git a/packages/provider/src/tasks/powCaptcha/powTasks.ts b/packages/provider/src/tasks/powCaptcha/powTasks.ts index 95af090fd6..c6098649eb 100644 --- a/packages/provider/src/tasks/powCaptcha/powTasks.ts +++ b/packages/provider/src/tasks/powCaptcha/powTasks.ts @@ -18,6 +18,7 @@ import { ProsopoEnvError, getLoggerDefault, } from "@prosopo/common"; +import type { Logger } from "@prosopo/common"; import { ApiParams, type CaptchaResult, @@ -30,20 +31,22 @@ import { } from "@prosopo/types"; import type { IProviderDatabase } from "@prosopo/types-database"; import { at, verifyRecency } from "@prosopo/util"; +import { computeFrictionlessScore } from "../frictionless/frictionlessTasksUtils.js"; import { checkPowSignature, validateSolution } from "./powTasksUtils.js"; -const logger = getLoggerDefault(); const DEFAULT_POW_DIFFICULTY = 4; export class PowCaptchaManager { pair: KeyringPair; db: IProviderDatabase; POW_SEPARATOR: string; + logger: Logger; - constructor(pair: KeyringPair, db: IProviderDatabase) { + constructor(pair: KeyringPair, db: IProviderDatabase, logger?: Logger) { this.pair = pair; this.db = db; this.POW_SEPARATOR = POW_SEPARATOR; + this.logger = logger || getLoggerDefault(); } /** @@ -52,6 +55,7 @@ export class PowCaptchaManager { * @param {string} userAccount - user that is solving the captcha * @param {string} dappAccount - dapp that is requesting the captcha * @param origin - not currently used + * @param powDifficulty */ async getPowCaptchaChallenge( userAccount: string, @@ -121,7 +125,7 @@ export class PowCaptchaManager { await this.db.getPowCaptchaRecordByChallenge(challenge); if (!challengeRecord) { - logger.debug("No record of this challenge"); + this.logger.debug("No record of this challenge"); // no record of this challenge return false; } @@ -213,6 +217,21 @@ export class PowCaptchaManager { await this.db.markDappUserPoWCommitmentsChecked([ challengeRecord.challenge, ]); - return { verified: true, score: challengeRecord.score }; + + let score: number | undefined; + if (challengeRecord.frictionlessTokenId) { + const tokenRecord = await this.db.getFrictionlessTokenRecordByTokenId( + challengeRecord.frictionlessTokenId, + ); + if (tokenRecord) { + score = computeFrictionlessScore(tokenRecord?.scoreComponents); + this.logger.info({ + tscoreComponents: tokenRecord?.scoreComponents, + score: score, + }); + } + } + + return { verified: true, ...(score ? { score } : {}) }; } } diff --git a/packages/provider/src/tests/unit/tasks/imgCaptcha/imgCaptchaTasks.unit.test.ts b/packages/provider/src/tests/unit/tasks/imgCaptcha/imgCaptchaTasks.unit.test.ts index 089aae1d4e..70a5890398 100644 --- a/packages/provider/src/tests/unit/tasks/imgCaptcha/imgCaptchaTasks.unit.test.ts +++ b/packages/provider/src/tests/unit/tasks/imgCaptcha/imgCaptchaTasks.unit.test.ts @@ -24,11 +24,11 @@ import { type Captcha, type CaptchaSolution, CaptchaStatus, - type PendingCaptchaRequest, type RequestHeaders, } from "@prosopo/types"; import type { IProviderDatabase, + PendingCaptchaRequest, UserCommitment, } from "@prosopo/types-database"; import { beforeEach, describe, expect, it, vi } from "vitest"; diff --git a/packages/provider/tsconfig.cjs.json b/packages/provider/tsconfig.cjs.json index 5c59c1456a..4441940fe6 100644 --- a/packages/provider/tsconfig.cjs.json +++ b/packages/provider/tsconfig.cjs.json @@ -11,6 +11,12 @@ "./src/**/*.tsx" ], "references": [ + { + "path": "../api-route/tsconfig.cjs.json" + }, + { + "path": "../api-express-router/tsconfig.cjs.json" + }, { "path": "../common/tsconfig.cjs.json" }, @@ -44,9 +50,6 @@ }, { "path": "../user-access-policy/tsconfig.cjs.json" - }, - { - "path": "../api-route/tsconfig.cjs.json" } ] } diff --git a/packages/provider/tsconfig.json b/packages/provider/tsconfig.json index 375beaa66a..7b92669d45 100644 --- a/packages/provider/tsconfig.json +++ b/packages/provider/tsconfig.json @@ -7,6 +7,12 @@ }, "include": ["src", "src/**/*.json"], "references": [ + { + "path": "../api-route" + }, + { + "path": "../api-express-router" + }, { "path": "../common" }, @@ -40,9 +46,6 @@ }, { "path": "../user-access-policy" - }, - { - "path": "../api-route" } ] } diff --git a/packages/types-database/src/index.ts b/packages/types-database/src/index.ts index 936b18d54f..6ac84e4283 100644 --- a/packages/types-database/src/index.ts +++ b/packages/types-database/src/index.ts @@ -12,3 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. export * from "./types/index.js"; +export type { PendingCaptchaRequest } from "./provider/pendingCaptchaRequest.js"; diff --git a/packages/types-database/src/provider/pendingCaptchaRequest.ts b/packages/types-database/src/provider/pendingCaptchaRequest.ts new file mode 100644 index 0000000000..11954c3153 --- /dev/null +++ b/packages/types-database/src/provider/pendingCaptchaRequest.ts @@ -0,0 +1,27 @@ +// Copyright 2021-2024 Prosopo (UK) Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +import { ApiParams, type RequestHeaders } from "@prosopo/types"; +import type { FrictionlessTokenId } from "../types/index.js"; + +export interface PendingCaptchaRequest { + accountId: string; + pending: boolean; + salt: string; + [ApiParams.requestHash]: string; + deadlineTimestamp: number; // unix timestamp + requestedAtTimestamp: number; // unix timestamp + ipAddress: bigint; + headers: RequestHeaders; + frictionlessTokenId?: FrictionlessTokenId; +} diff --git a/packages/types-database/src/types/provider.ts b/packages/types-database/src/types/provider.ts index d5ebf78930..d716e7456f 100644 --- a/packages/types-database/src/types/provider.ts +++ b/packages/types-database/src/types/provider.ts @@ -29,7 +29,6 @@ import { type Hash, type IUserData, type Item, - type PendingCaptchaRequest, type PoWCaptchaUser, type PoWChallengeComponents, type PoWChallengeId, @@ -50,11 +49,13 @@ import { bigint, boolean, nativeEnum, - number, object, string, + union, type infer as zInfer, + instanceof as zInstanceof, } from "zod"; +import type { PendingCaptchaRequest } from "../provider/pendingCaptchaRequest.js"; import { UserSettingsSchema } from "./client.js"; import type { IDatabase } from "./mongo.js"; @@ -101,7 +102,7 @@ export interface StoredCaptcha { serverChecked: boolean; storedAtTimestamp?: Timestamp; lastUpdatedTimestamp?: Timestamp; - score?: number; + frictionlessTokenId?: FrictionlessTokenId; } export interface UserCommitment extends Commit, StoredCaptcha { @@ -131,8 +132,12 @@ export const UserCommitmentSchema = object({ storedAtTimestamp: TimestampSchema.optional(), requestedAtTimestamp: TimestampSchema, lastUpdatedTimestamp: TimestampSchema.optional(), - score: number().optional(), -}) satisfies ZodType; + frictionlessTokenId: union([string(), zInstanceof(mongoose.Types.ObjectId)]) + .refine((val) => { + return mongoose.Types.ObjectId.isValid(val); + }) + .optional(), +}); export interface SolutionRecord extends CaptchaSolution { datasetId: string; @@ -196,6 +201,10 @@ export const PoWCaptchaRecordSchema = new Schema({ userSubmitted: { type: Boolean, required: true }, serverChecked: { type: Boolean, required: true }, storedAtTimestamp: { type: Date, required: false, expires: ONE_MONTH }, + frictionlessTokenId: { + type: mongoose.Schema.Types.ObjectId, + required: false, + }, }); // Set an index on the captchaId field, ascending @@ -226,6 +235,10 @@ export const UserCommitmentRecordSchema = new Schema({ storedAtTimestamp: { type: Number, required: false }, requestedAtTimestamp: { type: Number, required: true }, lastUpdatedTimestamp: { type: Number, required: false }, + frictionlessTokenId: { + type: mongoose.Schema.Types.ObjectId, + required: false, + }, }); // Set an index on the commitment id field, descending UserCommitmentRecordSchema.index({ id: -1 }); @@ -296,6 +309,8 @@ export type PendingCaptchaRequestMongoose = Omit< requestedAtTimestamp: Date; }; +export type FrictionlessTokenId = mongoose.Schema.Types.ObjectId; + export const PendingRecordSchema = new Schema({ accountId: { type: String, required: true }, pending: { type: Boolean, required: true }, @@ -305,7 +320,10 @@ export const PendingRecordSchema = new Schema({ requestedAtTimestamp: { type: Date, required: true, expires: ONE_WEEK }, ipAddress: { type: BigInt, required: true }, headers: { type: Object, required: true }, - score: { type: Number, required: false }, + frictionlessTokenId: { + type: mongoose.Types.ObjectId, + required: false, + }, }); // Set an index on the requestHash field, descending PendingRecordSchema.index({ requestHash: -1 }); @@ -349,12 +367,19 @@ ScheduledTaskRecordSchema.index({ processName: 1 }); ScheduledTaskRecordSchema.index({ processName: 1, status: 1 }); ScheduledTaskRecordSchema.index({ _id: 1, status: 1 }); -export type FrictionlessToken = { +export interface ScoreComponents { + baseScore: number; + lScore?: number; + timeout?: number; + accessPolicy?: number; +} + +export interface FrictionlessToken { token: string; score: number; threshold: number; - lScore?: number; -}; + scoreComponents: ScoreComponents; +} export type FrictionlessTokenRecord = mongoose.Document & FrictionlessToken; @@ -367,7 +392,12 @@ export const FrictionlessTokenRecordSchema = token: { type: String, required: true, unique: true }, score: { type: Number, required: true }, threshold: { type: Number, required: true }, - lScore: { type: Number, required: false }, + scoreComponents: { + baseScore: { type: Number, required: true }, + lScore: { type: Number, required: false }, + timeout: { type: Number, required: false }, + accessPolicy: { type: Number, required: false }, + }, createdAt: { type: Date, default: Date.now, expires: ONE_DAY }, }); @@ -376,7 +406,7 @@ FrictionlessTokenRecordSchema.index({ token: 1 }, { unique: true }); export type Session = { sessionId: string; createdAt: Date; - tokenId: ObjectId; + tokenId: FrictionlessTokenId; captchaType: CaptchaType; }; @@ -485,7 +515,7 @@ export interface IProviderDatabase extends IDatabase { requestedAtTimestamp: number, ipAddress: bigint, headers: RequestHeaders, - score?: number, + frictionlessTokenId?: FrictionlessTokenId, ): Promise; getPendingImageCommitment( @@ -584,7 +614,7 @@ export interface IProviderDatabase extends IDatabase { providerSignature: string, ipAddress: bigint, headers: RequestHeaders, - score?: number, + frictionlessTokenId?: FrictionlessTokenId, serverChecked?: boolean, userSubmitted?: boolean, userSignature?: string, @@ -610,8 +640,13 @@ export interface IProviderDatabase extends IDatabase { tokenRecord: FrictionlessToken, ): Promise; + updateFrictionlessTokenRecord( + tokenId: FrictionlessTokenId, + updates: Partial, + ): Promise; + getFrictionlessTokenRecordByTokenId( - tokenId: ObjectId, + tokenId: FrictionlessTokenId, ): Promise; getFrictionlessTokenRecordByToken( diff --git a/packages/types/src/provider/api.ts b/packages/types/src/provider/api.ts index 00afa206cf..5adad592e3 100644 --- a/packages/types/src/provider/api.ts +++ b/packages/types/src/provider/api.ts @@ -20,7 +20,6 @@ import { type ZodObject, type ZodOptional, array, - bigint, boolean, coerce, type input, @@ -216,18 +215,6 @@ export const VerifySolutionBody = object({ export type VerifySolutionBodyTypeInput = input; export type VerifySolutionBodyTypeOutput = output; -export interface PendingCaptchaRequest { - accountId: string; - pending: boolean; - salt: string; - [ApiParams.requestHash]: string; - deadlineTimestamp: number; // unix timestamp - requestedAtTimestamp: number; // unix timestamp - ipAddress: bigint; - headers: RequestHeaders; - score?: number; -} - export interface UpdateProviderClientsResponse extends ApiResponse { message: string; } @@ -243,11 +230,11 @@ export interface ApiResponse { export interface VerificationResponse extends ApiResponse { [ApiParams.verified]: boolean; + [ApiParams.score]?: number; } export interface ImageVerificationResponse extends VerificationResponse { [ApiParams.commitmentId]?: Hash; - [ApiParams.score]?: number; } export interface GetPowCaptchaResponse extends ApiResponse { diff --git a/packages/user-access-policy/src/rules/api/insertMany/apiInsertManyRulesArgsSchema.ts b/packages/user-access-policy/src/rules/api/insertMany/apiInsertManyRulesArgsSchema.ts index bd8565d983..4ffad0e955 100644 --- a/packages/user-access-policy/src/rules/api/insertMany/apiInsertManyRulesArgsSchema.ts +++ b/packages/user-access-policy/src/rules/api/insertMany/apiInsertManyRulesArgsSchema.ts @@ -14,13 +14,15 @@ import { boolean, object, string } from "zod"; import { ruleConfigSchema } from "../../rule/config/ruleConfigSchema.js"; -import { ruleIpSchema } from "../../rule/ip/ruleIpSchema.js"; const apiInsertManyRulesArgsSchema = object({ isUserBlocked: boolean(), clientId: string().optional(), description: string().optional(), - userIps: ruleIpSchema.array().optional(), + userIps: object({ + v4: string().array().optional(), + v6: string().array().optional(), + }), userIds: string().array().optional(), config: ruleConfigSchema.optional(), }); diff --git a/packages/user-access-policy/src/rules/api/insertMany/apiInsertManyRulesEndpoint.ts b/packages/user-access-policy/src/rules/api/insertMany/apiInsertManyRulesEndpoint.ts index 5ca594645b..6b0277fab5 100644 --- a/packages/user-access-policy/src/rules/api/insertMany/apiInsertManyRulesEndpoint.ts +++ b/packages/user-access-policy/src/rules/api/insertMany/apiInsertManyRulesEndpoint.ts @@ -17,7 +17,9 @@ import { type ApiEndpointResponse, ApiEndpointResponseStatus, } from "@prosopo/api-route"; +import { Address4, Address6 } from "ip-address"; import type { z } from "zod"; +import { ruleIpSchema } from "../../rule/ip/ruleIpSchema.js"; import type { Rule } from "../../rule/rule.js"; import type { RulesStorage } from "../../storage/rulesStorage.js"; import { @@ -56,9 +58,30 @@ class ApiInsertManyRulesEndpoint const userIps = args.userIps || []; - for (const userIp of userIps) { + for (const userIp of userIps.v4 || []) { + const ipAddress = new Address4(userIp); rules.push({ - userIp: userIp, + userIp: ruleIpSchema.parse({ + v4: { + asNumeric: ipAddress.bigInt(), + asString: ipAddress.address, + }, + }), + isUserBlocked: args.isUserBlocked, + description: args.description, + clientId: args.clientId, + config: args.config, + }); + } + for (const userIp of userIps.v6 || []) { + const ipAddress = new Address6(userIp); + rules.push({ + userIp: ruleIpSchema.parse({ + v4: { + asNumeric: ipAddress.bigInt(), + asString: ipAddress.address, + }, + }), isUserBlocked: args.isUserBlocked, description: args.description, clientId: args.clientId, diff --git a/packages/user-access-policy/src/rules/mongoose/rulesMongooseStorage.ts b/packages/user-access-policy/src/rules/mongoose/rulesMongooseStorage.ts index d97ab596f1..b60234d435 100644 --- a/packages/user-access-policy/src/rules/mongoose/rulesMongooseStorage.ts +++ b/packages/user-access-policy/src/rules/mongoose/rulesMongooseStorage.ts @@ -41,7 +41,11 @@ class RulesMongooseStorage implements RulesStorage { throw this.modelNotSetProsopoError(); } - const document = await this.writingModel.create(record); + let document = await this.writingModel.findOneAndUpdate(record); + + if (!document) { + document = await this.writingModel.create(record); + } const ruleRecord = this.convertMongooseRecordToRuleRecord( document.toObject(), @@ -55,6 +59,19 @@ class RulesMongooseStorage implements RulesStorage { throw this.modelNotSetProsopoError(); } + // Delete the existing records to avoid duplicates. + await this.writingModel.bulkWrite( + records.map((record) => ({ + deleteOne: { + filter: { + userId: record.userId, + clientId: record.clientId, + userIp: record.userIp, + } as Pick, + }, + })), + ); + const documents = await this.writingModel.insertMany(records); const objectDocuments = documents.map((document) => document.toObject());