From ea28c374a7681627f2a87aae9285eddf70d998a6 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 28 Jan 2025 23:29:02 +0100 Subject: [PATCH 01/12] feat(api): filter secrets by metadata --- backend/src/lib/api-docs/constants.ts | 4 ++- backend/src/server/routes/v3/secret-router.ts | 35 +++++++++++++++++++ .../secret-v2-bridge/secret-v2-bridge-dal.ts | 29 +++++++++++++++ .../secret-v2-bridge-types.ts | 5 +++ backend/src/services/secret/secret-service.ts | 7 ++++ backend/src/services/secret/secret-types.ts | 4 +++ 6 files changed, 83 insertions(+), 1 deletion(-) diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index eca11e9833..d38f9e59a7 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -688,7 +688,9 @@ export const RAW_SECRETS = { environment: "The slug of the environment to list secrets from.", secretPath: "The secret path to list secrets from.", includeImports: "Weather to include imported secrets or not.", - tagSlugs: "The comma separated tag slugs to filter secrets." + tagSlugs: "The comma separated tag slugs to filter secrets.", + secretMetadata: + "The secret metadata key-value pairs to filter secrets by. When querying for multiple metadata pairs, the query is treated as an AND operation. Secret metadata format is key1:value1,key2:value2." }, CREATE: { secretName: "The name of the secret to create.", diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 67fd09ac8e..0053a9ed26 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -181,6 +181,40 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], querystring: z.object({ + secretMetadata: z + .string() + .optional() + .transform((value) => { + if (!value) return undefined; + + const metadata = value.split(",").map((el) => { + const [key, val] = el.split(":"); + return { key, value: val }; + }); + + return metadata; + }) + .superRefine((el, ctx) => { + if (el && !Array.isArray(el)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Invalid secretMetadata format. Correct format is key1:value1,key2:value2" + }); + } + + if (el) { + for (const item of el) { + if (!item.key || !item.value) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "Invalid secretMetadata format, key or value is missing. Correct format is key1:value1,key2:value2" + }); + } + } + } + }) + .describe(RAW_SECRETS.LIST.secretMetadata), workspaceId: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceId), workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug), environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment), @@ -281,6 +315,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, projectId: workspaceId, path: secretPath, + secretMetadata: req.query.secretMetadata, includeImports: req.query.include_imports, recursive: req.query.recursive, tagSlugs: req.query.tagSlugs diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index 64ad77debc..77bf8c140c 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -414,6 +414,22 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { `${TableName.SecretTag}.id` ) .leftJoin(TableName.ResourceMetadata, `${TableName.SecretV2}.id`, `${TableName.ResourceMetadata}.secretId`) + .where((bd) => { + if (filters?.secretMetadata && filters.secretMetadata?.length > 0) { + filters.secretMetadata.forEach((meta) => { + void bd.whereExists((qb) => { + void qb + .select("secretId") + .from(TableName.ResourceMetadata) + .whereRaw(`"${TableName.ResourceMetadata}"."secretId" = "${TableName.SecretV2}"."id"`) + .where({ + [`${TableName.ResourceMetadata}.key` as string]: meta.key, + [`${TableName.ResourceMetadata}.value` as string]: meta.value + }); + }); + }); + } + }) .select( selectAllTableCols(TableName.SecretV2), db.raw( @@ -481,6 +497,19 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { } ] }); + + // if (secretMetadata) { + // return data.filter((s) => { + // if (!s.secretMetadata.length) return false; + + // return secretMetadata.every((m) => { + // const secretMeta = s.secretMetadata.find((sm) => sm.key === m.key); + // if (!secretMeta) return false; + // return secretMeta.value === m.value; + // }); + // }); + // } + return data; } catch (error) { throw new DatabaseError({ error, name: "get all secret" }); diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index 5e5cc26cd7..ee997c970f 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -30,6 +30,10 @@ export type TGetSecretsDTO = { includeImports?: boolean; recursive?: boolean; tagSlugs?: string[]; + secretMetadata?: { + key: string; + value: string; + }[]; orderBy?: SecretsOrderBy; orderDirection?: OrderByDirection; offset?: number; @@ -310,6 +314,7 @@ export type TFindSecretsByFolderIdsFilter = { orderDirection?: OrderByDirection; search?: string; tagSlugs?: string[]; + secretMetadata?: { key: string; value: string }[]; includeTagsInSearch?: boolean; keys?: string[]; }; diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index a9bba106c0..eb488c6a6e 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -1263,6 +1263,13 @@ export const secretServiceFactory = ({ name: "bot_not_found_error" }); + if (paramsV2.secretMetadata) { + throw new BadRequestError({ + message: "Please upgrade your project to filter secrets by metadata", + name: "SecretMetadataNotSupported" + }); + } + const { secrets, imports } = await getSecrets({ actorId, projectId, diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index d1011bf8cc..a857c0d305 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -182,6 +182,10 @@ export type TGetSecretsRawDTO = { includeImports?: boolean; recursive?: boolean; tagSlugs?: string[]; + secretMetadata?: { + key: string; + value: string; + }[]; orderBy?: SecretsOrderBy; orderDirection?: OrderByDirection; offset?: number; From ba03fc256b33b322f2392eb70759675a979ad3ca Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 28 Jan 2025 23:30:28 +0100 Subject: [PATCH 02/12] Update secret-router.ts --- backend/src/server/routes/v3/secret-router.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 0053a9ed26..cd928d4db7 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -194,16 +194,23 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { return metadata; }) - .superRefine((el, ctx) => { - if (el && !Array.isArray(el)) { + .superRefine((metadata, ctx) => { + if (metadata && !Array.isArray(metadata)) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Invalid secretMetadata format. Correct format is key1:value1,key2:value2" }); } - if (el) { - for (const item of el) { + if (metadata) { + if (metadata.length > 10) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "You can only filter by up to 10 metadata fields" + }); + } + + for (const item of metadata) { if (!item.key || !item.value) { ctx.addIssue({ code: z.ZodIssueCode.custom, From 55b0dc7f813d65cff4b2890d21a6db35c27e5e6a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Tue, 28 Jan 2025 23:35:07 +0100 Subject: [PATCH 03/12] chore: cleanup --- .../secret-v2-bridge/secret-v2-bridge-dal.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index 77bf8c140c..de758e3448 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -498,18 +498,6 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { ] }); - // if (secretMetadata) { - // return data.filter((s) => { - // if (!s.secretMetadata.length) return false; - - // return secretMetadata.every((m) => { - // const secretMeta = s.secretMetadata.find((sm) => sm.key === m.key); - // if (!secretMeta) return false; - // return secretMeta.value === m.value; - // }); - // }); - // } - return data; } catch (error) { throw new DatabaseError({ error, name: "get all secret" }); From d24a5d96e3ded8ebced12f85f58a6ad8c93f6a68 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Wed, 29 Jan 2025 14:24:23 +0100 Subject: [PATCH 04/12] requested changes --- .../secret-v2-bridge/secret-v2-bridge-dal.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index de758e3448..e42617d7e3 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -414,18 +414,12 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { `${TableName.SecretTag}.id` ) .leftJoin(TableName.ResourceMetadata, `${TableName.SecretV2}.id`, `${TableName.ResourceMetadata}.secretId`) - .where((bd) => { - if (filters?.secretMetadata && filters.secretMetadata?.length > 0) { + .where((qb) => { + if (filters?.secretMetadata && filters.secretMetadata.length > 0) { filters.secretMetadata.forEach((meta) => { - void bd.whereExists((qb) => { - void qb - .select("secretId") - .from(TableName.ResourceMetadata) - .whereRaw(`"${TableName.ResourceMetadata}"."secretId" = "${TableName.SecretV2}"."id"`) - .where({ - [`${TableName.ResourceMetadata}.key` as string]: meta.key, - [`${TableName.ResourceMetadata}.value` as string]: meta.value - }); + void qb.where({ + [`${TableName.ResourceMetadata}.key` as string]: meta.key, + [`${TableName.ResourceMetadata}.value` as string]: meta.value }); }); } From 0968893d4b37af6d24f0929de57760aa96de9cdd Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Thu, 30 Jan 2025 21:41:17 +0100 Subject: [PATCH 05/12] improved filtering format --- backend/src/server/routes/v3/secret-router.ts | 39 ++++++++++++++----- .../secret-v2-bridge/secret-v2-bridge-dal.ts | 4 +- .../secret-v2-bridge-types.ts | 6 +-- backend/src/services/secret/secret-types.ts | 4 +- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index cd928d4db7..ca326367c1 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -184,15 +184,36 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { secretMetadata: z .string() .optional() - .transform((value) => { - if (!value) return undefined; + .transform((val) => { + if (!val) return undefined; - const metadata = value.split(",").map((el) => { - const [key, val] = el.split(":"); - return { key, value: val }; - }); + const result: { key?: string; value?: string }[] = []; + const pairs = val.split(","); - return metadata; + for (const pair of pairs) { + const pairResult: { key?: string; value?: string } = {}; + const parts = pair.split(/[:=]/).map((part) => part.trim()); + + for (let i = 0; i < parts.length - 1; i += 1) { + const current = parts[i].toLowerCase(); + const next = parts[i + 1]; + + if (current === "key" && next) { + pairResult.key = next; + } else if (current === "value" && next) { + pairResult.value = next; + } + } + + // Only add pair if at least one of key or value is present + if (pairResult.key || pairResult.value) { + result.push(pairResult); + } + } + + if (!result.length) return undefined; + + return result; }) .superRefine((metadata, ctx) => { if (metadata && !Array.isArray(metadata)) { @@ -211,11 +232,11 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } for (const item of metadata) { - if (!item.key || !item.value) { + if (!item.key && !item.value) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: - "Invalid secretMetadata format, key or value is missing. Correct format is key1:value1,key2:value2" + "Invalid secretMetadata format, key or value must be provided. Correct format is key1:value1,key2:value2" }); } } diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index e42617d7e3..fb271cfdcd 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -418,8 +418,8 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { if (filters?.secretMetadata && filters.secretMetadata.length > 0) { filters.secretMetadata.forEach((meta) => { void qb.where({ - [`${TableName.ResourceMetadata}.key` as string]: meta.key, - [`${TableName.ResourceMetadata}.value` as string]: meta.value + ...(meta.key ? { [`${TableName.ResourceMetadata}.key` as string]: meta.key } : {}), + ...(meta.value ? { [`${TableName.ResourceMetadata}.value` as string]: meta.value } : {}) }); }); } diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index ee997c970f..f69ef52be2 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -31,8 +31,8 @@ export type TGetSecretsDTO = { recursive?: boolean; tagSlugs?: string[]; secretMetadata?: { - key: string; - value: string; + key?: string; + value?: string; }[]; orderBy?: SecretsOrderBy; orderDirection?: OrderByDirection; @@ -314,7 +314,7 @@ export type TFindSecretsByFolderIdsFilter = { orderDirection?: OrderByDirection; search?: string; tagSlugs?: string[]; - secretMetadata?: { key: string; value: string }[]; + secretMetadata?: { key?: string; value?: string }[]; includeTagsInSearch?: boolean; keys?: string[]; }; diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index a857c0d305..887f62d25f 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -183,8 +183,8 @@ export type TGetSecretsRawDTO = { recursive?: boolean; tagSlugs?: string[]; secretMetadata?: { - key: string; - value: string; + key?: string; + value?: string; }[]; orderBy?: SecretsOrderBy; orderDirection?: OrderByDirection; From be99e40050c54fdc318cedd18397a3680abb6bc5 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 3 Feb 2025 12:54:54 +0400 Subject: [PATCH 06/12] fix: improved filter --- backend/src/server/routes/v3/secret-router.ts | 75 +++++++++++-------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index ca326367c1..62de9d69ba 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -181,68 +181,81 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], querystring: z.object({ - secretMetadata: z + filters: z .string() .optional() .transform((val) => { + // TODO: Add support for more filtering fields & operators. + // Currently only supports secretMetadata filtering in a specific format to allow for further expansion of filtering in the future. + if (!val) return undefined; + const secretMetadataSection = [val].find((section) => section.startsWith("secretMetadata=")); + + if (!secretMetadataSection) return undefined; + + const metadataContent = secretMetadataSection.replace("secretMetadata=", ""); - const result: { key?: string; value?: string }[] = []; - const pairs = val.split(","); + const secretMetadataResult: { key?: string; value?: string }[] = []; + let currentPair: { key?: string; value?: string } = {}; - for (const pair of pairs) { - const pairResult: { key?: string; value?: string } = {}; - const parts = pair.split(/[:=]/).map((part) => part.trim()); + const parts = metadataContent.split(/(? { - if (metadata && !Array.isArray(metadata)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: "Invalid secretMetadata format. Correct format is key1:value1,key2:value2" - }); - } + .superRefine((filters, ctx) => { + if (filters?.secretMetadata) { + if (!Array.isArray(filters.secretMetadata)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "Invalid secretMetadata format in filters. Correct format is filters=secretMetadata=key=key1,value=val1" + }); + } - if (metadata) { - if (metadata.length > 10) { + if (filters.secretMetadata.length > 10) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: "You can only filter by up to 10 metadata fields" }); } - for (const item of metadata) { + for (const item of filters.secretMetadata) { if (!item.key && !item.value) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: - "Invalid secretMetadata format, key or value must be provided. Correct format is key1:value1,key2:value2" + "Invalid secretMetadata format in filters, key or value must be provided. Correct format is filters=secretMetadata=key=key1,value=val1" }); } } } }) .describe(RAW_SECRETS.LIST.secretMetadata), + workspaceId: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceId), workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug), environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment), @@ -343,7 +356,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, projectId: workspaceId, path: secretPath, - secretMetadata: req.query.secretMetadata, + secretMetadata: req.query.filters?.secretMetadata, includeImports: req.query.include_imports, recursive: req.query.recursive, tagSlugs: req.query.tagSlugs From 9d976de19bca932cdeb906876f76cf145282819b Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 3 Feb 2025 21:13:47 +0400 Subject: [PATCH 07/12] Revert "fix: improved filter" This reverts commit be99e40050c54fdc318cedd18397a3680abb6bc5. --- backend/src/server/routes/v3/secret-router.ts | 75 ++++++++----------- 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 62de9d69ba..ca326367c1 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -181,81 +181,68 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], querystring: z.object({ - filters: z + secretMetadata: z .string() .optional() .transform((val) => { - // TODO: Add support for more filtering fields & operators. - // Currently only supports secretMetadata filtering in a specific format to allow for further expansion of filtering in the future. - if (!val) return undefined; - const secretMetadataSection = [val].find((section) => section.startsWith("secretMetadata=")); - - if (!secretMetadataSection) return undefined; - - const metadataContent = secretMetadataSection.replace("secretMetadata=", ""); - const secretMetadataResult: { key?: string; value?: string }[] = []; - let currentPair: { key?: string; value?: string } = {}; + const result: { key?: string; value?: string }[] = []; + const pairs = val.split(","); - const parts = metadataContent.split(/(? part.trim()); - for (const part of parts) { - const [type, value] = part.split("="); + for (let i = 0; i < parts.length - 1; i += 1) { + const current = parts[i].toLowerCase(); + const next = parts[i + 1]; - if (type === "key") { - if (currentPair.key || currentPair.value) { - secretMetadataResult.push(currentPair); - currentPair = {}; + if (current === "key" && next) { + pairResult.key = next; + } else if (current === "value" && next) { + pairResult.value = next; } - currentPair.key = value; - } else if (type === "value") { - currentPair.value = value; - secretMetadataResult.push(currentPair); - currentPair = {}; } - } - if (currentPair.key || currentPair.value) { - secretMetadataResult.push(currentPair); + // Only add pair if at least one of key or value is present + if (pairResult.key || pairResult.value) { + result.push(pairResult); + } } - if (!secretMetadataResult.length) return undefined; + if (!result.length) return undefined; - return { - secretMetadata: secretMetadataResult - }; + return result; }) - .superRefine((filters, ctx) => { - if (filters?.secretMetadata) { - if (!Array.isArray(filters.secretMetadata)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: - "Invalid secretMetadata format in filters. Correct format is filters=secretMetadata=key=key1,value=val1" - }); - } + .superRefine((metadata, ctx) => { + if (metadata && !Array.isArray(metadata)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Invalid secretMetadata format. Correct format is key1:value1,key2:value2" + }); + } - if (filters.secretMetadata.length > 10) { + if (metadata) { + if (metadata.length > 10) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: "You can only filter by up to 10 metadata fields" }); } - for (const item of filters.secretMetadata) { + for (const item of metadata) { if (!item.key && !item.value) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: - "Invalid secretMetadata format in filters, key or value must be provided. Correct format is filters=secretMetadata=key=key1,value=val1" + "Invalid secretMetadata format, key or value must be provided. Correct format is key1:value1,key2:value2" }); } } } }) .describe(RAW_SECRETS.LIST.secretMetadata), - workspaceId: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceId), workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug), environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment), @@ -356,7 +343,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, projectId: workspaceId, path: secretPath, - secretMetadata: req.query.filters?.secretMetadata, + secretMetadata: req.query.secretMetadata, includeImports: req.query.include_imports, recursive: req.query.recursive, tagSlugs: req.query.tagSlugs From 407fd8eda7795da937eda5227ee9c2de32efba4a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 3 Feb 2025 21:16:07 +0400 Subject: [PATCH 08/12] chore: rename to metadata filter --- backend/src/server/routes/v3/secret-router.ts | 4 ++-- .../src/services/secret-v2-bridge/secret-v2-bridge-types.ts | 4 ++-- backend/src/services/secret/secret-service.ts | 2 +- backend/src/services/secret/secret-types.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index ca326367c1..4155b0d11e 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -181,7 +181,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { } ], querystring: z.object({ - secretMetadata: z + metadataFilter: z .string() .optional() .transform((val) => { @@ -343,7 +343,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, projectId: workspaceId, path: secretPath, - secretMetadata: req.query.secretMetadata, + secretMetadata: req.query.metadataFilter, includeImports: req.query.include_imports, recursive: req.query.recursive, tagSlugs: req.query.tagSlugs diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index f69ef52be2..37512d93ad 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -30,7 +30,7 @@ export type TGetSecretsDTO = { includeImports?: boolean; recursive?: boolean; tagSlugs?: string[]; - secretMetadata?: { + metadataFilter?: { key?: string; value?: string; }[]; @@ -314,7 +314,7 @@ export type TFindSecretsByFolderIdsFilter = { orderDirection?: OrderByDirection; search?: string; tagSlugs?: string[]; - secretMetadata?: { key?: string; value?: string }[]; + metadataFilter?: { key?: string; value?: string }[]; includeTagsInSearch?: boolean; keys?: string[]; }; diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index eb488c6a6e..d0f92ef575 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -1263,7 +1263,7 @@ export const secretServiceFactory = ({ name: "bot_not_found_error" }); - if (paramsV2.secretMetadata) { + if (paramsV2.metadataFilter) { throw new BadRequestError({ message: "Please upgrade your project to filter secrets by metadata", name: "SecretMetadataNotSupported" diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 887f62d25f..a0082efc67 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -182,7 +182,7 @@ export type TGetSecretsRawDTO = { includeImports?: boolean; recursive?: boolean; tagSlugs?: string[]; - secretMetadata?: { + metadataFilter?: { key?: string; value?: string; }[]; From cd5078d8b72877c288437f1e3cf9d5a6d685f540 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 3 Feb 2025 23:22:20 +0400 Subject: [PATCH 09/12] Update secret-router.ts --- backend/src/server/routes/v3/secret-router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 4155b0d11e..e691e74170 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -343,7 +343,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, projectId: workspaceId, path: secretPath, - secretMetadata: req.query.metadataFilter, + metadataFilter: req.query.metadataFilter, includeImports: req.query.include_imports, recursive: req.query.recursive, tagSlugs: req.query.tagSlugs From 071f37666e83b6a8f32c9fa358920dead198d452 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Mon, 3 Feb 2025 23:22:27 +0400 Subject: [PATCH 10/12] Update secret-v2-bridge-dal.ts --- backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index fb271cfdcd..27a62a6d3c 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -415,8 +415,8 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { ) .leftJoin(TableName.ResourceMetadata, `${TableName.SecretV2}.id`, `${TableName.ResourceMetadata}.secretId`) .where((qb) => { - if (filters?.secretMetadata && filters.secretMetadata.length > 0) { - filters.secretMetadata.forEach((meta) => { + if (filters?.metadataFilter && filters.metadataFilter.length > 0) { + filters.metadataFilter.forEach((meta) => { void qb.where({ ...(meta.key ? { [`${TableName.ResourceMetadata}.key` as string]: meta.key } : {}), ...(meta.value ? { [`${TableName.ResourceMetadata}.value` as string]: meta.value } : {}) From 8d7b5968d394002dd1ef2e977d1bac2939170548 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Thu, 6 Feb 2025 07:39:47 +0400 Subject: [PATCH 11/12] requested changes --- backend/src/server/routes/v3/secret-router.ts | 29 +++++++++---------- .../secret-v2-bridge/secret-v2-bridge-dal.ts | 10 +++++-- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index e691e74170..84227b59e1 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -188,32 +188,29 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { if (!val) return undefined; const result: { key?: string; value?: string }[] = []; - const pairs = val.split(","); + const pairs = val.split("|"); for (const pair of pairs) { - const pairResult: { key?: string; value?: string } = {}; - const parts = pair.split(/[:=]/).map((part) => part.trim()); + const keyValuePair: { key?: string; value?: string } = {}; + const parts = pair.split(/[,=]/); - for (let i = 0; i < parts.length - 1; i += 1) { - const current = parts[i].toLowerCase(); - const next = parts[i + 1]; + for (let i = 0; i < parts.length; i += 2) { + const identifier = parts[i].trim().toLowerCase(); + const value = parts[i + 1]?.trim(); - if (current === "key" && next) { - pairResult.key = next; - } else if (current === "value" && next) { - pairResult.value = next; + if (identifier === "key" && value) { + keyValuePair.key = value; + } else if (identifier === "value" && value) { + keyValuePair.value = value; } } - // Only add pair if at least one of key or value is present - if (pairResult.key || pairResult.value) { - result.push(pairResult); + if (keyValuePair.key && keyValuePair.value) { + result.push(keyValuePair); } } - if (!result.length) return undefined; - - return result; + return result.length ? result : undefined; }) .superRefine((metadata, ctx) => { if (metadata && !Array.isArray(metadata)) { diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index 27a62a6d3c..99980fba77 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -417,9 +417,13 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { .where((qb) => { if (filters?.metadataFilter && filters.metadataFilter.length > 0) { filters.metadataFilter.forEach((meta) => { - void qb.where({ - ...(meta.key ? { [`${TableName.ResourceMetadata}.key` as string]: meta.key } : {}), - ...(meta.value ? { [`${TableName.ResourceMetadata}.value` as string]: meta.value } : {}) + void qb.whereExists((subQuery) => { + void subQuery + .select("secretId") + .from(TableName.ResourceMetadata) + .whereRaw(`"${TableName.ResourceMetadata}"."secretId" = "${TableName.SecretV2}"."id"`) + .where(`${TableName.ResourceMetadata}.key`, meta.key) + .where(`${TableName.ResourceMetadata}.value`, meta.value); }); }); } From 70c9761abe861aeabdd38c5b3dfe03ff5669c7c3 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Fri, 7 Feb 2025 07:49:42 +0400 Subject: [PATCH 12/12] requested changes --- backend/src/lib/api-docs/constants.ts | 4 ++-- backend/src/server/routes/v3/secret-router.ts | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index d38f9e59a7..8e79d6be8c 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -689,8 +689,8 @@ export const RAW_SECRETS = { secretPath: "The secret path to list secrets from.", includeImports: "Weather to include imported secrets or not.", tagSlugs: "The comma separated tag slugs to filter secrets.", - secretMetadata: - "The secret metadata key-value pairs to filter secrets by. When querying for multiple metadata pairs, the query is treated as an AND operation. Secret metadata format is key1:value1,key2:value2." + metadataFilter: + "The secret metadata key-value pairs to filter secrets by. When querying for multiple metadata pairs, the query is treated as an AND operation. Secret metadata format is key=value1,value=value2|key=value3,value=value4." }, CREATE: { secretName: "The name of the secret to create.", diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 84227b59e1..e30cfcede4 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -216,7 +216,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { if (metadata && !Array.isArray(metadata)) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "Invalid secretMetadata format. Correct format is key1:value1,key2:value2" + message: + "Invalid secretMetadata format. Correct format is key=value1,value=value2|key=value3,value=value4." }); } @@ -233,13 +234,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { ctx.addIssue({ code: z.ZodIssueCode.custom, message: - "Invalid secretMetadata format, key or value must be provided. Correct format is key1:value1,key2:value2" + "Invalid secretMetadata format, key or value must be provided. Correct format is key=value1,value=value2|key=value3,value=value4." }); } } } }) - .describe(RAW_SECRETS.LIST.secretMetadata), + .describe(RAW_SECRETS.LIST.metadataFilter), workspaceId: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceId), workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug), environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment),