diff --git a/Makefile b/Makefile index 11bcb9faae..9099bf60e4 100644 --- a/Makefile +++ b/Makefile @@ -371,7 +371,7 @@ STERN_VERSION = v2.6.1 CHART_TESTING_VERSION = v3.10.1 K3D_IMAGE = docker.io/rancher/k3s:v1.28.6-k3s2 TESTS = [nginx,api,features-kubernetes,bulk-deployment,features-kubernetes-2,features-variables,active-standby-kubernetes,tasks,drush,python,gitlab,github,bitbucket,services,workflows] -CHARTS_TREEISH = prerelease/lagoon_v218 +CHARTS_TREEISH = keycloak-migrations TASK_IMAGES = task-activestandby # Symlink the installed kubectl client if the correct version is already diff --git a/docker-compose.yaml b/docker-compose.yaml index 04e8c03b81..03a6e862b6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -43,7 +43,7 @@ services: - ./node-packages:/app/node-packages:delegated environment: - CONSOLE_LOGGING_LEVEL=trace - api-init: + api-db-init: image: ${IMAGE_REPO:-lagoon}/api command: > sh -c "./node_modules/.bin/knex migrate:list --cwd /app/services/api/database @@ -53,8 +53,24 @@ services: - ./node-packages:/app/node-packages:delegated - /app/node-packages/commons/dist depends_on: - - api-db - - keycloak + api-db: + condition: service_started + keycloak: + condition: service_started + api-lagoon-migrations: + image: ${IMAGE_REPO:-lagoon}/api + command: sh -c "./node_modules/.bin/tsc && node -r dotenv-extended/config dist/migrations/lagoon/migration.js" + volumes: + - ./services/api/src:/app/services/api/src + environment: + - KEYCLOAK_URL=http://172.17.0.1:8088 + - NODE_ENV=development + - CONSOLE_LOGGING_LEVEL=trace + depends_on: + api-db-init: + condition: service_completed_successfully # don't start the lagoon migrations until the db migrations is completed + keycloak: + condition: service_started api: image: ${IMAGE_REPO:-lagoon}/api command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index" @@ -74,8 +90,10 @@ services: - S3_BAAS_SECRET_ACCESS_KEY=minio123 - CONSOLE_LOGGING_LEVEL=debug depends_on: - - api-init - - keycloak + api-lagoon-migrations: + condition: service_started + keycloak: + condition: service_started ports: - '3000:3000' # Uncomment for local new relic tracking diff --git a/services/api/database/migrations/20240312000000_group_projects.js b/services/api/database/migrations/20240312000000_group_projects.js new file mode 100644 index 0000000000..3c7e3602da --- /dev/null +++ b/services/api/database/migrations/20240312000000_group_projects.js @@ -0,0 +1,40 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function(knex) { + group_projects = await knex.schema.hasTable('kc_group_projects'); + if (!group_projects) { + return knex.schema + // this table holds the main group to organization id association + .createTable('kc_group_organization', function (table) { + table.increments('id').notNullable().primary(); + table.string('group_id', 50).notNullable(); + table.integer('organization_id').notNullable(); + table.unique(['group_id', 'organization_id'], {indexName: 'group_organization'}); + }) + // this table holds the main group to organization id association + .createTable('kc_group_projects', function (table) { + table.increments('id').notNullable().primary(); + table.string('group_id', 50).notNullable(); + table.integer('project_id').notNullable(); + table.unique(['group_id', 'project_id'], {indexName: 'group_project'}); + }) + } + else { + return knex.schema + } +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function(knex) { + // caveats around this are that the rollback can only work while data is still saved in keycloak attributes + // once we remove that duplication of attribute into keycloak, this rollback would result in data loss for group>project associations + // for any group project associations made after the attribute removal + return knex.schema + .dropTable('kc_group_organization') + .dropTable('kc_group_projects') +}; diff --git a/services/api/package.json b/services/api/package.json index 3e51649d85..5a19eedc9b 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -15,7 +15,8 @@ "sync:gitlab:all": "yarn run sync:gitlab:users && yarn run sync:gitlab:groups && yarn run sync:gitlab:projects", "sync:opendistro-security": "node --max-http-header-size=80000 dist/helpers/sync-groups-opendistro-security", "sync:bitbucket:repo-permissions": "node dist/bitbucket-sync/repo-permissions", - "sync:harbor:projects": "node dist/migrations/2-harbor/harborSync.js" + "sync:harbor:projects": "node dist/migrations/2-harbor/harborSync.js", + "migrations:lagoon": "node dist/migrations/lagoon/migrations.js" }, "keywords": [], "author": "amazee.io (http://www.amazee.io)", @@ -49,7 +50,7 @@ "jsonwebtoken": "^8.0.1", "keycloak-admin": "https://github.com/amazeeio/keycloak-admin.git#bd015d2e34634f262c0827f00620657427e3c252", "keycloak-connect": "^5.0.0", - "knex": "^0.95.15", + "knex": "^3.0.1", "mariadb": "^2.5.2", "moment": "^2.24.0", "morgan": "^1.9.0", diff --git a/services/api/src/apolloServer.js b/services/api/src/apolloServer.js index a2594f10ff..95c4e7f5bd 100644 --- a/services/api/src/apolloServer.js +++ b/services/api/src/apolloServer.js @@ -120,28 +120,6 @@ const apolloServer = new ApolloServer({ esClient, }; - // get all keycloak groups, do this early to reduce the number of times this is called otherwise - // but doing this early and once is pretty cheap - let keycloakGroups = [] - try { - // check redis for the allgroups cache value - const data = await getRedisKeycloakCache("allgroups"); - let buff = new Buffer(data, 'base64'); - keycloakGroups = JSON.parse(buff.toString('utf-8')); - } catch (err) { - logger.warn(`Couldn't check redis keycloak cache: ${err.message}`); - // if it can't be recalled from redis, get the data from keycloak - const allGroups = await Group.Group(modelClients).loadAllGroups(); - keycloakGroups = await Group.Group(modelClients).transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } - } - let currentUser = {}; let serviceAccount = {}; // if this is a user request, get the users keycloak groups too, do this one to reduce the number of times it is called elsewhere @@ -151,11 +129,12 @@ const apolloServer = new ApolloServer({ const keycloakGrant = grant let legacyGrant = legacyCredentials ? legacyCredentials : null if (keycloakGrant) { + // get all the users keycloak groups, do this early to reduce the number of times this is called otherwise keycloakUsersGroups = await User.User(modelClients).getAllGroupsForUser(keycloakGrant.access_token.content.sub); serviceAccount = await keycloakGrantManager.obtainFromClientCredentials(); currentUser = await User.User(modelClients).loadUserById(keycloakGrant.access_token.content.sub); // grab the users project ids and roles in the first request - groupRoleProjectIds = await User.User(modelClients).getAllProjectsIdsForUser(currentUser, keycloakUsersGroups); + groupRoleProjectIds = await User.User(modelClients).getAllProjectsIdsForUser(currentUser.id, keycloakUsersGroups); } return { @@ -173,7 +152,6 @@ const apolloServer = new ApolloServer({ ProjectModel: ProjectModel.ProjectModel(modelClients), EnvironmentModel: EnvironmentModel.EnvironmentModel(modelClients) }, - keycloakGroups, keycloakUsersGroups, }; }, @@ -206,28 +184,6 @@ const apolloServer = new ApolloServer({ esClient, }; - // get all keycloak groups, do this early to reduce the number of times this is called otherwise - // but doing this early and once is pretty cheap - let keycloakGroups = [] - try { - // check redis for the allgroups cache value - const data = await getRedisKeycloakCache("allgroups"); - let buff = new Buffer(data, 'base64'); - keycloakGroups = JSON.parse(buff.toString('utf-8')); - } catch (err) { - logger.warn(`Couldn't check redis keycloak cache: ${err.message}`); - // if it can't be recalled from redis, get the data from keycloak - const allGroups = await Group.Group(modelClients).loadAllGroups(); - keycloakGroups = await Group.Group(modelClients).transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } - } - let currentUser = {}; let serviceAccount = {}; // if this is a user request, get the users keycloak groups too, do this one to reduce the number of times it is called elsewhere @@ -237,11 +193,12 @@ const apolloServer = new ApolloServer({ const keycloakGrant = req.kauth ? req.kauth.grant : null let legacyGrant = req.legacyCredentials ? req.legacyCredentials : null if (keycloakGrant) { + // get all the users keycloak groups, do this early to reduce the number of times this is called otherwise keycloakUsersGroups = await User.User(modelClients).getAllGroupsForUser(keycloakGrant.access_token.content.sub); serviceAccount = await keycloakGrantManager.obtainFromClientCredentials(); currentUser = await User.User(modelClients).loadUserById(keycloakGrant.access_token.content.sub); // grab the users project ids and roles in the first request - groupRoleProjectIds = await User.User(modelClients).getAllProjectsIdsForUser(currentUser, keycloakUsersGroups); + groupRoleProjectIds = await User.User(modelClients).getAllProjectsIdsForUser(currentUser.id, keycloakUsersGroups); } // do a permission check to see if the user is platform admin/owner, or has permission for `viewAll` on certain resources @@ -308,7 +265,6 @@ const apolloServer = new ApolloServer({ ProjectModel: ProjectModel.ProjectModel(modelClients), EnvironmentModel: EnvironmentModel.EnvironmentModel(modelClients) }, - keycloakGroups, keycloakUsersGroups, adminScopes: { projectViewAll: projectViewAll, diff --git a/services/api/src/clients/redisClient.ts b/services/api/src/clients/redisClient.ts index 270603ab71..d2556db4b7 100644 --- a/services/api/src/clients/redisClient.ts +++ b/services/api/src/clients/redisClient.ts @@ -27,6 +27,8 @@ redisClient.on('error', function(error) { console.error(error); }); +export const groupCacheExpiry = toNumber(getConfigFromEnv('REDIS_GROUP_CACHE_EXPIRY', '172800')) + export const get = promisify(redisClient.get).bind(redisClient); const hgetall = promisify(redisClient.hgetall).bind(redisClient); const smembers = promisify(redisClient.smembers).bind(redisClient); diff --git a/services/api/src/migrations/lagoon/migration.ts b/services/api/src/migrations/lagoon/migration.ts new file mode 100644 index 0000000000..595cc575d0 --- /dev/null +++ b/services/api/src/migrations/lagoon/migration.ts @@ -0,0 +1,30 @@ +import { waitForKeycloak } from '../../util/waitForKeycloak'; +import { envHasConfig } from '../../util/config'; +import { logger } from '../../loggers/logger'; +import { migrate } from '../../util/db' + + +(async () => { + await waitForKeycloak(); + + // run any migrations that need keycloak before starting the api + try { + // run the migrations + logger.info('previous migrations:'); + const before = await migrate.migrate.list(); + for (const l of before) { + if (l.length) { + logger.info(`- ${l.name}`) + } + } + logger.info('performing migrations if required'); + // actually run the migrations + await migrate.migrate.latest(); + logger.info('migrations completed'); + } catch (e) { + logger.fatal(`Couldn't run migrations: ${e.message}`); + process.exit(1) + } + + process.exit() +})(); diff --git a/services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts b/services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts new file mode 100644 index 0000000000..4a32137977 --- /dev/null +++ b/services/api/src/migrations/lagoon/migrations/20240312000000_group_projects.ts @@ -0,0 +1,54 @@ +import * as R from 'ramda'; +import { logger } from '@lagoon/commons/dist/logs/local-logger'; +import { getKeycloakAdminClient } from '../../../clients/keycloak-admin'; +import { sqlClientPool } from '../../../clients/sqlClient'; +import { esClient } from '../../../clients/esClient'; +import redisClient from '../../../clients/redisClient'; +import { Group } from '../../../models/group'; +import { Helpers } from '../../../resources/group/helpers'; + +export const up = async (migrate) => { + const keycloakAdminClient = await getKeycloakAdminClient(); + + const GroupModel = Group({ + sqlClientPool, + keycloakAdminClient, + esClient, + redisClient + }); + + // load all groups from keycloak + const allGroups = await GroupModel.loadAllGroups(); + // flatten them out + const flattenGroups = (groups, group) => { + groups.push(R.omit(['subGroups'], group)); + const flatSubGroups = group.subGroups.reduce(flattenGroups, []); + return groups.concat(flatSubGroups); + }; + const fgs = R.pipe( + R.reduce(flattenGroups, []), + )(allGroups) + // loop over the groups ignoring `role-subgroup` groups + for (const fg of fgs) { + if (fg.attributes['type'] != "role-subgroup") { + const groupProjects = await GroupModel.getProjectsFromGroup(fg); + for (const pid of groupProjects) { + logger.info(`Migrating project ${pid} and group ${fg.name}/${fg.id} to database`) + // add the project group association to the database + await Helpers(sqlClientPool).addProjectToGroup(pid, fg.id) + } + // if the group is in an organization + if (R.prop('lagoon-organization', fg.attributes)) { + // add the organization group association to the database + logger.info(`Migrating group ${fg.name}/${fg.id} in organization ${R.prop('lagoon-organization', fg.attributes)} to database`) + await Helpers(sqlClientPool).addOrganizationToGroup(parseInt(R.prop('lagoon-organization', fg.attributes)[0], 10), fg.id) + } + } + } + + return migrate.schema +} + +export const down = async (migrate) => { + return migrate.schema +} diff --git a/services/api/src/models/group.ts b/services/api/src/models/group.ts index 5cb760ff3a..bad9303a6d 100644 --- a/services/api/src/models/group.ts +++ b/services/api/src/models/group.ts @@ -9,8 +9,9 @@ import { logger } from '../loggers/logger'; // @ts-ignore import GroupRepresentation from 'keycloak-admin/lib/defs/groupRepresentation'; import { User } from './user'; -import { saveRedisKeycloakCache, get, del, redisClient } from '../clients/redisClient'; +import { groupCacheExpiry, get, del, redisClient } from '../clients/redisClient'; import { Helpers as projectHelpers } from '../resources/project/helpers'; +import { Helpers as groupHelpers } from '../resources/group/helpers'; import { sqlClientPool } from '../clients/sqlClient'; import { log } from 'winston'; @@ -35,6 +36,7 @@ export interface Group { // All subgroups according to keycloak. subGroups?: GroupRepresentation[]; attributes?: IGroupAttributes; + realmRoles?: any[]; } interface GroupMembership { @@ -160,33 +162,48 @@ export const Group = (clients: { }; const loadGroupById = async (id: string): Promise => { + let group; - let keycloakGroup: Group - keycloakGroup = await keycloakAdminClient.groups.findOne({ - id, - briefRepresentation: false, - }); - if (R.isNil(keycloakGroup)) { - throw new GroupNotFoundError(`Group not found: ${id}`); + // check if the group is in the cache + try { + group = await getGroupCacheById(id) + } catch(err) { + logger.warn(`Error reading redis, falling back to direct lookup: ${err.message}`); } + // if not in the cache, check keycloak + if (!group) { + let keycloakGroup: Group + keycloakGroup = await keycloakAdminClient.groups.findOne({ + id, + briefRepresentation: false, + }); + if (R.isNil(keycloakGroup)) { + throw new GroupNotFoundError(`Group not found: ${id}`); + } - const groups = await transformKeycloakGroups([keycloakGroup]); - keycloakGroup = groups[0] - - return keycloakGroup; + const groups = await transformKeycloakGroups([keycloakGroup]); + keycloakGroup = groups[0] + // then save the entry in the cache + try { + await saveGroupCache(keycloakGroup) + } catch (err) { + logger.info (`Error saving redis: ${err.message}`) + } + return keycloakGroup; + } + return group; }; const loadGroupByName = async (name: string): Promise => { - const cacheKey = `cache:keycloak:group-id:${name}`; - let groupId; + let group; try { - groupId = await get(cacheKey); + group = await getGroupCacheByName(name) } catch(err) { - logger.info(`Error reading redis ${cacheKey}: ${err.message}`); + logger.warn(`Error reading redis, falling back to direct lookup: ${err.message}`); } - if (!groupId) { + if (!group) { const keycloakGroups = await keycloakAdminClient.groups.find({ search: name }); @@ -202,28 +219,30 @@ export const Group = (clients: { return groups.concat(flatSubGroups); }; - groupId = R.pipe( + group = R.pipe( R.reduce(flattenGroups, []), - R.filter(R.propEq('name', name)), - R.path(['0', 'id']) - )(keycloakGroups) as string; + R.filter(R.propEq('name', name)) + )(keycloakGroups); - if (R.isNil(groupId)) { + if (R.isEmpty(group)) { throw new GroupNotFoundError(`Group not found: ${name}`); } + const keycloakGroup = await keycloakAdminClient.groups.findOne({ + id: group[0].id, + briefRepresentation: false, + }); + const groups = await transformKeycloakGroups([keycloakGroup]); + group = groups[0]; try { - await redisClient.multi() - .set(cacheKey, groupId) - .expire(cacheKey, 172800) // 48 hours - .exec(); + await saveGroupCache(group) } catch (err) { - logger.info (`Error saving redis ${cacheKey}: ${err.message}`) + logger.info (`Error saving redis: ${err.message}`) } } // @ts-ignore - return await loadGroupById(groupId); + return group; }; const loadGroupByIdOrName = async ( @@ -244,9 +263,10 @@ export const Group = (clients: { // briefRepresentation pulls all the group information from keycloak including the attributes // this means we don't need to iterate over all the groups one by one anymore to get the full group information const fullGroups = await keycloakAdminClient.groups.find({briefRepresentation: false}); - const keycloakGroups = await transformKeycloakGroups(fullGroups); - - return keycloakGroups; + // no need to transform, just return the full response, only the `allGroups` and `deleteAllGroups` resolvers use this + // and the `sync-groups-opendistro-security` consumption of this helper sync script is going to + // go away in the future when we move to the `lagoon-opensearch-sync` supporting service + return fullGroups; }; const loadParentGroup = async (groupInput: Group): Promise => @@ -449,33 +469,73 @@ export const Group = (clients: { } }; + // return only project ids that still exist in lagoon in the response for which projects this group has assigned + // in the past some groups could have been deleted from lagoon and their `attribute` in keycloak remained + const getProjectsFromGroup = async ( + group: Group + ): Promise => { + try { + const groupProjectIds = getProjectIdsFromGroup(group); + // remove deleted projects from the result to prevent null errors in user queries + const existingProjects = await projectHelpers(sqlClientPool).getAllProjectsIn(groupProjectIds); + let existingProjectsIds = []; + existingProjectsIds.push(...existingProjects.map(epi => epi.id)); + return existingProjectsIds + } catch (err) { + return []; + } + }; + const getGroupMembership = async ( group: Group ): Promise => { const UserModel = User(clients); const roleSubgroups = group.subGroups.filter(isRoleSubgroup); - let membership = []; - for (const roleSubgroup of roleSubgroups) { - const keycloakUsers = await keycloakAdminClient.groups.listMembers({ - id: roleSubgroup.id, - briefRepresentation: false, - }); - - let members = []; - for (const keycloakUser of keycloakUsers) { - const users = await UserModel.transformKeycloakUsers([keycloakUser]); - const fullUser = users[0] - const member = { - user: fullUser, - role: roleSubgroup.realmRoles[0], - roleSubgroupId: roleSubgroup.id - }; - - members = [...members, member]; + // check for members cache + const membersCacheKey = `cache:keycloak:group-members:${group.id}`; + try { + let data = await get(membersCacheKey); + if (data) { + let buff = Buffer.from(data, 'base64'); + // set membership to cached data + membership = JSON.parse(buff.toString('utf-8')); } + } catch(err) { + logger.warn(`Error reading redis ${membersCacheKey}, falling back to direct lookup: ${err.message}`); + } + if (membership.length == 0) { + for (const roleSubgroup of roleSubgroups) { + const keycloakUsers = await keycloakAdminClient.groups.listMembers({ + id: roleSubgroup.id, + briefRepresentation: false, + }); - membership = [...membership, ...members]; + let members = []; + for (const keycloakUser of keycloakUsers) { + const users = await UserModel.transformKeycloakUsers([keycloakUser]); + const fullUser = users[0] + const member = { + user: fullUser, + role: roleSubgroup.realmRoles[0], + roleSubgroupId: roleSubgroup.id + }; + + members = [...members, member]; + } + + membership = [...membership, ...members]; + } + // save latest members cache + const data = Buffer.from(JSON.stringify(membership)).toString('base64') + try { + await redisClient.multi() + .set(membersCacheKey, data) + .expire(membersCacheKey, groupCacheExpiry) // 48 hours + .exec(); + } catch (err) { + logger.info (`Error saving redis ${membersCacheKey}: ${err.message}`) + } } return membership; @@ -496,7 +556,7 @@ export const Group = (clients: { return membership; }; - const addGroup = async (groupInput: Group): Promise => { + const addGroup = async (groupInput: Group, projectId?: number, organizationId?: number): Promise => { // Don't allow duplicate subgroup names try { const existingGroup = await loadGroupByName(groupInput.name); @@ -530,6 +590,12 @@ export const Group = (clients: { } const group = await loadGroupById(response.id); + if (projectId) { + await groupHelpers(sqlClientPool).addProjectToGroup(projectId, group.id) + } + if (organizationId) { + await groupHelpers(sqlClientPool).addOrganizationToGroup(organizationId, group.id) + } // Set the parent group if (R.prop('parentGroupId', groupInput)) { @@ -562,16 +628,6 @@ export const Group = (clients: { } } - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } - return group; }; @@ -579,6 +635,7 @@ export const Group = (clients: { const oldGroup = await loadGroupById(groupInput.id); try { + await purgeGroupCache(oldGroup) await keycloakAdminClient.groups.update( { id: groupInput.id @@ -612,16 +669,6 @@ export const Group = (clients: { }); } } - - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } return newGroup; }; @@ -631,7 +678,8 @@ export const Group = (clients: { id, briefRepresentation: false, }); - await del(`cache:keycloak:group-id:${keycloakGroup.name}`); + await groupHelpers(sqlClientPool).deleteGroup(id) + await purgeGroupCache(keycloakGroup) await keycloakAdminClient.groups.del({ id }); } catch (err) { if (err.response.status && err.response.status === 404) { @@ -640,16 +688,6 @@ export const Group = (clients: { throw new Error(`Error deleting group ${id}: ${err}`); } } - - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } }; const addUserToGroup = async ( @@ -690,16 +728,7 @@ export const Group = (clients: { } catch (err) { throw new Error(`Could not add user to group: ${err.message}`); } - - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } + await purgeGroupCache(group) return await loadGroupById(group.id); }; @@ -707,6 +736,8 @@ export const Group = (clients: { user: User, group: Group ): Promise => { + // purge the caches to ensure current data + await purgeGroupCache(group, true) const members = await getGroupMembership(group); const userMembership = R.find(R.pathEq(['user', 'id'], user.id))(members); @@ -722,19 +753,9 @@ export const Group = (clients: { } catch (err) { throw new Error(`Could not remove user from group: ${err.message}`); } - + // purge after removing the user to ensure current data + await purgeGroupCache(group) } - - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } - return await loadGroupById(group.id); }; @@ -755,6 +776,7 @@ export const Group = (clients: { } try { + await groupHelpers(sqlClientPool).addProjectToGroup(projectId, group.id) await keycloakAdminClient.groups.update( { id: group.id @@ -768,21 +790,13 @@ export const Group = (clients: { } } ); + // purge the caches to ensure current data + await purgeGroupCache(group) } catch (err) { throw new Error( `Error setting projects for group ${group.name}: ${err.message}` ); } - - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } }; const removeProjectFromGroup = async ( @@ -798,6 +812,7 @@ export const Group = (clients: { )(groupProjectIds); try { + await groupHelpers(sqlClientPool).removeProjectFromGroup(projectId, group.id) await keycloakAdminClient.groups.update( { id: group.id @@ -811,21 +826,13 @@ export const Group = (clients: { } } ); + // purge the caches to ensure current data + await purgeGroupCache(group) } catch (err) { throw new Error( `Error setting projects for group ${group.name}: ${err.message}` ); } - - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } }; // helper to remove project from groups @@ -857,23 +864,14 @@ export const Group = (clients: { } } ); + // purge the caches to ensure current data + await purgeGroupCache(group) } catch (err) { throw new Error( `Error setting projects for group ${group.name}: ${err.message}` ); } } - - // once the project is remove from the groups, update the cache - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); - } }; // helper to remove user from groups @@ -883,6 +881,8 @@ export const Group = (clients: { ): Promise => { for (const g in groups) { const group = groups[g] + // purge the caches to ensure current data + await purgeGroupCache(group, true) const members = await getGroupMembership(group); const userMembership = R.find(R.pathEq(['user', 'id'], user.id))(members); @@ -898,16 +898,8 @@ export const Group = (clients: { throw new Error(`Could not remove user from group: ${err.message}`); } } - } - - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); + // purge the caches to ensure current data + await purgeGroupCache(group) } }; @@ -916,6 +908,8 @@ export const Group = (clients: { group: Group, project: String, ): Promise => { + // purge the caches to ensure current data + await purgeGroupCache(group, true) const members = await getGroupMembership(group); for (const u in members) { @@ -932,21 +926,55 @@ export const Group = (clients: { } } } + // purge the caches to ensure current data + await purgeGroupCache(group) + return await loadGroupById(group.id); + }; - // once the users are removed from the group, update the cache - const allGroups = await loadAllGroups(); - const keycloakGroups = await transformKeycloakGroups(allGroups); - const data = Buffer.from(JSON.stringify(keycloakGroups)).toString('base64') - try { - // then attempt to save it to redis - await saveRedisKeycloakCache("allgroups", data); - } catch (err) { - logger.warn(`Couldn't save redis keycloak cache: ${err.message}`); + const purgeGroupCache = async (group: Group, membersOnly: Boolean=false): Promise => { + if (!membersOnly) { + await del(`cache:keycloak:group-id:${group.name}`); + await del(`cache:keycloak:group:${group.id}`); } + await del(`cache:keycloak:group-members:${group.id}`); + }; - return await loadGroupById(group.id); + const saveGroupCache = async (group: Group): Promise => { + const idCacheKey = `cache:keycloak:group:${group.id}`; + const nameCacheKey = `cache:keycloak:group-id:${group.name}`; + const data = Buffer.from(JSON.stringify(group)).toString('base64') + await redisClient.multi() + .set(nameCacheKey, group.id) + .expire(nameCacheKey, groupCacheExpiry) // 48 hours + .exec(); + await redisClient.multi() + .set(idCacheKey, data) + .expire(idCacheKey, groupCacheExpiry) // 48 hours + .exec(); }; + const getGroupCacheByName =async (name: String) => { + const nameCacheKey = `cache:keycloak:group-id:${name}`; + let group; + let groupId = await get(nameCacheKey); + group = await getGroupCacheById(groupId) + } + + const getGroupCacheById = async (groupId: String): Promise => { + let group; + const idCacheKey = `cache:keycloak:group:${groupId}`; + try { + let data = await get(idCacheKey); + if (data) { + let buff = Buffer.from(data, 'base64'); + group = JSON.parse(buff.toString('utf-8')); + return group + } + } catch(err) { + logger.warn(`Error reading redis ${idCacheKey}, falling back to direct lookup: ${err.message}`); + } + } + return { loadAllGroups, loadGroupById, @@ -960,6 +988,7 @@ export const Group = (clients: { loadGroupsByProjectIdFromGroups, getProjectsFromGroupAndParents, getProjectsFromGroupAndSubgroups, + getProjectsFromGroup, addGroup, updateGroup, deleteGroup, diff --git a/services/api/src/models/user.ts b/services/api/src/models/user.ts index 0606353a06..7064155b4c 100644 --- a/services/api/src/models/user.ts +++ b/services/api/src/models/user.ts @@ -9,6 +9,7 @@ import { sqlClientPool } from '../clients/sqlClient'; import { query } from '../util/db'; import { Sql } from '../resources/user/sql'; import { getConfigFromEnv } from '../util/config'; +import { Helpers as groupHelpers } from '../resources/group/helpers'; import { getRedisKeycloakCache } from '../clients/redisClient'; interface IUserAttributes { @@ -45,7 +46,7 @@ interface UserModel { loadUsersByOrganizationId: (organizationId: number) => Promise; getAllOrganizationIdsForUser: (userInput: User) => Promise; getAllGroupsForUser: (userId: string) => Promise; - getAllProjectsIdsForUser: (userInput: User, groups?: Group[]) => Promise<{}>; + getAllProjectsIdsForUser: (userId: string, groups?: Group[]) => Promise<{}>; getUserRolesForProject: ( userInput: User, projectId: number, @@ -318,134 +319,77 @@ export const User = (clients: { const getAllGroupsForUser = async (userId: string, organization?: number): Promise => { const GroupModel = Group(clients); - let groups = []; - const roleSubgroups = await keycloakAdminClient.users.listGroups({ id: userId, briefRepresentation: false }); - let fullGroups = []; - try { - // check redis for the allgroups cache value - const data = await getRedisKeycloakCache("allgroups"); - let buff = new Buffer(data, 'base64'); - fullGroups = JSON.parse(buff.toString('utf-8')); - } catch (err) { - logger.warn(`Couldn't check redis keycloak cache: ${err.message}`); - // if it can't be recalled from redis, get the data from keycloak - const allGroups = await GroupModel.loadAllGroups(); - fullGroups = await GroupModel.transformKeycloakGroups(allGroups); - } - const regexp = /-(owner|maintainer|developer|reporter|guest)$/g; - - for (const fullGroup of fullGroups) { - for (const roleSubgroup of roleSubgroups) { - for (const fullSubgroup of fullGroup.subGroups) { - if (roleSubgroup.name.replace(regexp, "") == fullSubgroup.name) { - let group = fullSubgroup - if (organization) { - if (group.attributes["lagoon-organization"] != organization) { - continue - } - } - let filtergroup = group.subGroups.filter((item) => item.name == roleSubgroup.name); - group.subGroups = filtergroup - groups.push(group) - } - } - if (roleSubgroup.name.replace(regexp, "") == fullGroup.name) { - let group = fullGroup - if (organization) { - if (group.attributes["lagoon-organization"] != organization) { - continue - } + let userGroups = []; + for (const ug of roleSubgroups) { + // push the group ids into an array of group ids only for sql lookups + let index = userGroups.findIndex((item) => item.name === ug.name.replace(regexp, "")); + if (index === -1) { + const parentGroup = await GroupModel.loadGroupByName(ug.name.replace(regexp, "")) + if (organization) { + const parentOrg = R.defaultTo('', R.prop('lagoon-organization', parentGroup.attributes)).toString() + const orgid = parentOrg.split(',')[0] + if (parseInt(orgid, 10) != organization) { + continue } - let filtergroup = group.subGroups.filter((item) => item.name == roleSubgroup.name); - group.subGroups = filtergroup - groups.push(group) } + userGroups.push(parentGroup); } } - const retGroups = await GroupModel.transformKeycloakGroups(groups); + const retGroups = await GroupModel.transformKeycloakGroups(userGroups); return retGroups; }; const getAllProjectsIdsForUser = async ( - userInput: User, + userId: string, groups?: Group[] - ): Promise<{}> => { + ): Promise<{}> => { const GroupModel = Group(clients); - let roleProjectIds = {}; - - if (groups) { - // if groups are provided (eg, the groups have previously been calculated in a prior step), then process those groups here and extract the project ids from them - for (const roleSubgroup of groups) { - for (const fullSubgroup of roleSubgroup.subGroups) { - // https://github.com/uselagoon/lagoon/pull/3358 references potential issue with the lagoon-projects attribute where there could be empty values - // getProjectsFromGroupAndSubgroups already covers this fix - const projectIds = await GroupModel.getProjectsFromGroupAndSubgroups( - roleSubgroup - ); - if (!roleProjectIds[fullSubgroup.realmRoles[0]]) { - roleProjectIds[fullSubgroup.realmRoles[0]] = [] - } - projectIds.forEach(pid => { - roleProjectIds[fullSubgroup.realmRoles[0]].indexOf(pid) === -1 ? roleProjectIds[fullSubgroup.realmRoles[0]].push(pid) : "" - }) - } - } - - return roleProjectIds; - } else { - // otherwise fall back to the previous method of getting groups and project ids which is an expensive call to keycloak if repeated often - const roleSubgroups = await keycloakAdminClient.users.listGroups({ - id: userInput.id, + let userGroups = []; + if (!groups) { + groups = await keycloakAdminClient.users.listGroups({ + id: userId, briefRepresentation: false }); + } - const fullGroups = await keycloakAdminClient.groups.find({briefRepresentation: false}); - - // currently in lagoon groups with a role will have the role as a prefix, this regix can be used to identify and remove it to get the parent group name - const regexp = /-(owner|maintainer|developer|reporter|guest)$/g; - - for (const fullGroup of fullGroups) { - for (const roleSubgroup of roleSubgroups) { - for (const fullSubgroup of fullGroup.subGroups) { - if (roleSubgroup.name.replace(regexp, "") == fullSubgroup.name) { - // https://github.com/uselagoon/lagoon/pull/3358 references potential issue with the lagoon-projects attribute where there could be empty values - // getProjectsFromGroupAndSubgroups already covers this fix - const projectIds = await GroupModel.getProjectsFromGroupAndSubgroups( - fullSubgroup - ); - if (!roleProjectIds[roleSubgroup.realmRoles[0]]) { - roleProjectIds[roleSubgroup.realmRoles[0]] = [] - } - projectIds.forEach(pid => { - roleProjectIds[roleSubgroup.realmRoles[0]].indexOf(pid) === -1 ? roleProjectIds[roleSubgroup.realmRoles[0]].push(pid) : "" - }) - } - } - if (roleSubgroup.name.replace(regexp, "") == fullGroup.name) { + const regexp = /-(owner|maintainer|developer|reporter|guest)$/g; + for (const ug of groups) { + // push the group ids into an array of group ids only for sql lookups + let index = userGroups.findIndex((item) => item.name === ug.name.replace(regexp, "")); + if (index === -1) { + const parentGroup = await GroupModel.loadGroupByName(ug.name.replace(regexp, "")) + userGroups.push(parentGroup); + } + } + let roleProjectIds = {}; + for (const roleSubgroup of userGroups) { + for (const fullSubgroup of groups) { + for (const group of fullSubgroup.subGroups) { + // filter out the users roles subgroup from the main group so the correct roles are attached to the project ids + if (roleSubgroup.name.replace(regexp, "") == fullSubgroup.name) { // https://github.com/uselagoon/lagoon/pull/3358 references potential issue with the lagoon-projects attribute where there could be empty values // getProjectsFromGroupAndSubgroups already covers this fix const projectIds = await GroupModel.getProjectsFromGroupAndSubgroups( - fullGroup + roleSubgroup ); - if (!roleProjectIds[roleSubgroup.realmRoles[0]]) { - roleProjectIds[roleSubgroup.realmRoles[0]] = [] + if (!roleProjectIds[group.realmRoles[0]]) { + roleProjectIds[group.realmRoles[0]] = [] } projectIds.forEach(pid => { - roleProjectIds[roleSubgroup.realmRoles[0]].indexOf(pid) === -1 ? roleProjectIds[roleSubgroup.realmRoles[0]].push(pid) : "" + roleProjectIds[group.realmRoles[0]].indexOf(pid) === -1 ? roleProjectIds[group.realmRoles[0]].push(pid) : "" }) } } } - - return roleProjectIds; } + return roleProjectIds; }; const getUserRolesForProject = async ( diff --git a/services/api/src/resources/deployment/resolvers.ts b/services/api/src/resources/deployment/resolvers.ts index 382977600c..6cf859f5df 100644 --- a/services/api/src/resources/deployment/resolvers.ts +++ b/services/api/src/resources/deployment/resolvers.ts @@ -136,9 +136,7 @@ export const getDeploymentsByBulkId: ResolverFn = async ( return []; } - const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser({ - id: keycloakGrant.access_token.content.sub - }, keycloakUsersGroups); + const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser(keycloakGrant.access_token.content.sub, keycloakUsersGroups); userProjectIds = getUserProjectIdsFromRoleProjectIds(userProjectRoles); } @@ -186,9 +184,7 @@ export const getDeploymentsByFilter: ResolverFn = async ( return []; } - const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser({ - id: keycloakGrant.access_token.content.sub - }, keycloakUsersGroups); + const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser(keycloakGrant.access_token.content.sub, keycloakUsersGroups); userProjectIds = getUserProjectIdsFromRoleProjectIds(userProjectRoles); } @@ -1316,9 +1312,7 @@ export const bulkDeployEnvironmentLatest: ResolverFn = async ( return []; } - const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser({ - id: keycloakGrant.access_token.content.sub - }, keycloakUsersGroups); + const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser(keycloakGrant.access_token.content.sub, keycloakUsersGroups); userProjectIds = getUserProjectIdsFromRoleProjectIds(userProjectRoles); } diff --git a/services/api/src/resources/environment/resolvers.ts b/services/api/src/resources/environment/resolvers.ts index 7dc8107c6f..e4ca6a5c54 100644 --- a/services/api/src/resources/environment/resolvers.ts +++ b/services/api/src/resources/environment/resolvers.ts @@ -260,9 +260,7 @@ export const getEnvironmentsByKubernetes: ResolverFn = async ( } // Only return projects the user can view - const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser({ - id: keycloakGrant.access_token.content.sub, - }, keycloakUsersGroups); + const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser(keycloakGrant.access_token.content.sub, keycloakUsersGroups); userProjectIds = getUserProjectIdsFromRoleProjectIds(userProjectRoles); } diff --git a/services/api/src/resources/fact/resolvers.ts b/services/api/src/resources/fact/resolvers.ts index e94dad24dd..48ee4dc54d 100644 --- a/services/api/src/resources/fact/resolvers.ts +++ b/services/api/src/resources/fact/resolvers.ts @@ -131,9 +131,7 @@ export const getProjectsByFactSearch: ResolverFn = async ( let userProjectIds: number[]; if (!adminScopes.projectViewAll) { - const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser({ - id: keycloakGrant.access_token.content.sub - }, keycloakUsersGroups); + const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser(keycloakGrant.access_token.content.sub, keycloakUsersGroups); userProjectIds = getUserProjectIdsFromRoleProjectIds(userProjectRoles); } @@ -151,9 +149,7 @@ export const getEnvironmentsByFactSearch: ResolverFn = async ( let userProjectIds: number[]; if (!adminScopes.projectViewAll) { - const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser({ - id: keycloakGrant.access_token.content.sub - }, keycloakUsersGroups); + const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser(keycloakGrant.access_token.content.sub, keycloakUsersGroups); userProjectIds = getUserProjectIdsFromRoleProjectIds(userProjectRoles); } diff --git a/services/api/src/resources/group/helpers.ts b/services/api/src/resources/group/helpers.ts new file mode 100644 index 0000000000..6fd80568e6 --- /dev/null +++ b/services/api/src/resources/group/helpers.ts @@ -0,0 +1,111 @@ +import * as R from 'ramda'; +import { Pool } from 'mariadb'; +import { asyncPipe } from '@lagoon/commons/dist/util/func'; +import { query } from '../../util/db'; +import { Sql } from './sql'; +import { logger } from '../../loggers/logger'; + +export const Helpers = (sqlClientPool: Pool) => { + return { + selectProjectIdsByGroupIDs: async (groupIds: string[]) => { + const projectIdsArray = await query( + sqlClientPool, + Sql.selectProjectIdsByGroupIDs(groupIds) + ) + if (projectIdsArray[0]["projectIds"] != null) { + const values = projectIdsArray[0]["projectIds"].split(','); + return values + } + return [] + }, + selectProjectIdsByGroupID: async (groupId: string) => { + const projectIdsArray = await query( + sqlClientPool, + Sql.selectProjectIdsByGroupID(groupId) + ) + if (projectIdsArray[0]["projectIds"] != null) { + const values = projectIdsArray[0]["projectIds"].split(','); + return values + } + return [] + }, + selectGroupsByProjectId: async (models, projectId: number) => { + const groupsArray = await query( + sqlClientPool, + Sql.selectGroupsByProjectId(projectId) + ) + if (groupsArray[0]["groupIds"] != null) { + const groups = groupsArray[0]["groupIds"].split(','); + let projectGroups = [] + // collect the groups now + for (const o in groups) { + projectGroups.push(await models.GroupModel.loadGroupById(groups[o])) + } + return projectGroups + } + return [] + }, + selectGroupsByOrganizationId: async (models, organizationId: number) => { + const groupsArray = await query( + sqlClientPool, + Sql.selectGroupsByOrganizationId(organizationId) + ) + if (groupsArray[0]["groupIds"] != null) { + const groups = groupsArray[0]["groupIds"].split(','); + let orgGroups = [] + // collect the groups now + for (const o in groups) { + orgGroups.push(await models.GroupModel.loadGroupById(groups[o])) + } + return orgGroups + } + return [] + }, + selectOrganizationByGroupId: async (groupId: string) => { + return await query( + sqlClientPool, + Sql.selectOrganizationByGroupId(groupId) + ) + }, + removeProjectFromGroup: async (projectId: number, groupId: string) => { + await query( + sqlClientPool, + Sql.removeProjectFromGroup(projectId, groupId) + ) + }, + addProjectToGroup: async (projectId: number, groupId: string) => { + try { + await query( + sqlClientPool, + Sql.addProjectToGroup({projectId, groupId}) + ); + } catch (err) { + return null + } + }, + addOrganizationToGroup: async (organizationId: number, groupId: string) => { + try { + await query( + sqlClientPool, + Sql.addOrganizationToGroup({organizationId, groupId}) + ); + } catch (err) { + return null + } + }, + deleteGroup: async (groupId: string) => { + await query( + sqlClientPool, + Sql.deleteOrganizationGroup(groupId) + ) + await query( + sqlClientPool, + Sql.deleteProjectGroup(groupId) + ) + try { + } catch (err) { + logger.info(`${err}`) + } + }, + }; +}; \ No newline at end of file diff --git a/services/api/src/resources/group/resolvers.ts b/services/api/src/resources/group/resolvers.ts index 8694649f3a..e9afb2cef9 100644 --- a/services/api/src/resources/group/resolvers.ts +++ b/services/api/src/resources/group/resolvers.ts @@ -10,13 +10,15 @@ import { Helpers as projectHelpers } from '../project/helpers'; import { OpendistroSecurityOperations } from './opendistroSecurity'; import { KeycloakUnauthorizedError } from '../../util/auth'; import { Helpers as organizationHelpers } from '../organization/helpers'; +import { Helpers } from './helpers'; +import { sqlClientPool } from '../../clients/sqlClient'; const DISABLE_NON_ORGANIZATION_GROUP_CREATION = process.env.DISABLE_NON_ORGANIZATION_GROUP_CREATION || "false" export const getAllGroups: ResolverFn = async ( root, { name, type }, - { hasPermission, models, keycloakGrant, keycloakGroups, keycloakUsersGroups, adminScopes } + { hasPermission, models, keycloakGrant, keycloakUsersGroups, adminScopes } ) => { // use the admin scope check instead of `hasPermission` for speed if (adminScopes.groupViewAll) { @@ -26,7 +28,8 @@ export const getAllGroups: ResolverFn = async ( const group = await models.GroupModel.loadGroupByName(name); return [group]; } else { - const groups = keycloakGroups; + const allGroups = await models.GroupModel.loadAllGroups(); + const groups = await models.GroupModel.transformKeycloakGroups(allGroups); const filterFn = (key, val) => group => group[key].includes(val); const filteredByName = groups.filter(filterFn('name', name)); const filteredByType = groups.filter(filterFn('type', type)); @@ -139,12 +142,12 @@ export const getGroupRolesByUserId: ResolverFn =async ( export const getMembersByGroupId: ResolverFn = async ( { id }, _input, - { hasPermission, models, keycloakGrant, keycloakGroups } + { hasPermission, models, keycloakGrant } ) => { try { // members resolver is only called by group, no need to check the permissions on the group // as the group resolver will have already checked permission - const group = await getGroupFromGroupsById(id, keycloakGroups); + const group = await models.GroupModel.loadGroupById(id); const members = await models.GroupModel.getGroupMembership(group); return members; } catch (err) { @@ -188,12 +191,12 @@ export const getMemberCountByGroupId: ResolverFn = async ( export const getGroupsByProjectId: ResolverFn = async ( { id: pid }, _input, - { hasPermission, sqlClientPool, models, keycloakGrant, keycloakGroups, keycloakUsersGroups, adminScopes } + { hasPermission, sqlClientPool, models, keycloakGrant, keycloakUsersGroups, adminScopes } ) => { - const projectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(pid, keycloakGroups); // use the admin scope check instead of `hasPermission` for speed if (adminScopes.groupViewAll) { try { + const projectGroups = await Helpers(sqlClientPool).selectGroupsByProjectId(models, pid) return projectGroups; } catch (err) { if (!keycloakGrant) { @@ -202,6 +205,7 @@ export const getGroupsByProjectId: ResolverFn = async ( } } } else { + const projectGroups = await Helpers(sqlClientPool).selectGroupsByProjectId(models, pid) const user = await models.UserModel.loadUserById( keycloakGrant.access_token.content.sub ); @@ -213,26 +217,20 @@ export const getGroupsByProjectId: ResolverFn = async ( const usersOrgsViewer = R.defaultTo('', R.prop('lagoon-organizations-viewer', user.attributes)).toString() if (usersOrgs != "" ) { - const usersOrgsArr = usersOrgs.split(','); - for (const userOrg of usersOrgsArr) { - const project = await projectHelpers(sqlClientPool).getProjectById(pid); - if (project.organization == userOrg) { - const orgGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(project.organization, keycloakGroups); - for (const pGroup of orgGroups) { - userGroups.push(pGroup) - } + const uOrgs = usersOrgs.split(','); + for (const userOrg of uOrgs) { + const orgGroups = await Helpers(sqlClientPool).selectGroupsByOrganizationId(models, userOrg) + for (const pGroup of orgGroups) { + userGroups.push(pGroup) } } } if (usersOrgsViewer != "" ) { - const usersOrgsArr = usersOrgsViewer.split(','); - for (const userOrg of usersOrgsArr) { - const project = await projectHelpers(sqlClientPool).getProjectById(pid); - if (project.organization == userOrg) { - const orgViewerGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(project.organization, keycloakGroups); - for (const pGroup of orgViewerGroups) { - userGroups.push(pGroup) - } + const uOrgs = usersOrgsViewer.split(','); + for (const userOrg of uOrgs) { + const orgGroups = await Helpers(sqlClientPool).selectGroupsByOrganizationId(models, userOrg) + for (const pGroup of orgGroups) { + userGroups.push(pGroup) } } } @@ -311,7 +309,7 @@ export const getGroupByName: ResolverFn = async ( export const addGroup: ResolverFn = async ( _root, { input }, - { models, sqlClientPool, keycloakGrant, adminScopes, hasPermission, userActivityLogger, keycloakGroups} + { models, sqlClientPool, keycloakGrant, adminScopes, hasPermission, userActivityLogger} ) => { let attributes = null; // check if this is a group being added in an organization @@ -326,7 +324,7 @@ export const addGroup: ResolverFn = async ( organization: input.organization }); - const orgGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(input.organization, keycloakGroups); + const orgGroups = await Helpers(sqlClientPool).selectGroupsByOrganizationId(models, input.organization) let groupCount = 0 for (const pGroup in orgGroups) { // project-default-groups don't count towards group quotas @@ -380,19 +378,11 @@ export const addGroup: ResolverFn = async ( name: input.name, parentGroupId, ...attributes, - }); + }, null, input.organization); await models.GroupModel.addProjectToGroup(null, group); - // if the user is not an admin, then add the user as an owner to the group - let userAlreadyHasAccess = false; - if (adminScopes.projectViewAll) { - userAlreadyHasAccess = true - } - // if the group is created without the addOrgOwner boolean set to true, then do not add the user to the group as its owner - if (!input.addOrgOwner) { - userAlreadyHasAccess = true - } - if (!userAlreadyHasAccess && keycloakGrant) { + // if the user is not an admin, or an organization add, then add the user as an owner to the group + if (!adminScopes.projectViewAll && !input.organization && keycloakGrant) { const user = await models.UserModel.loadUserById( keycloakGrant.access_token.content.sub ); @@ -515,11 +505,12 @@ export const deleteGroup: ResolverFn = async ( export const deleteAllGroups: ResolverFn = async ( _root, _args, - { models, hasPermission, keycloakGroups } + { models, hasPermission } ) => { await hasPermission('group', 'deleteAll'); - const groups = keycloakGroups; + const allGroups = await models.GroupModel.loadAllGroups(); + const groups = await models.GroupModel.transformKeycloakGroups(allGroups); let deleteErrors: String[] = []; for (const group of groups) { @@ -722,7 +713,7 @@ export const getAllProjectsByGroupId: ResolverFn = async ( export const getAllProjectsInGroup: ResolverFn = async ( _root, { input: groupInput }, - { models, sqlClientPool, hasPermission, keycloakGrant, keycloakGroups, keycloakUsersGroups, adminScopes } + { models, sqlClientPool, hasPermission, keycloakGrant, adminScopes } ) => { const { GroupModel: { loadGroupByIdOrName, getProjectsFromGroupAndSubgroups } @@ -732,13 +723,7 @@ export const getAllProjectsInGroup: ResolverFn = async ( if (adminScopes.groupViewAll) { try { // get group from all keycloak groups apollo context - let group = []; - if (groupInput.name) { - group = await getGroupFromGroupsByName(groupInput.name, keycloakGroups); - } - if (groupInput.id) { - group = await getGroupFromGroupsById(groupInput.id, keycloakGroups); - } + const group = await loadGroupByIdOrName(groupInput); const projectIdsArray = await getProjectsFromGroupAndSubgroups(group); return projectIdsArray.map(async id => projectHelpers(sqlClientPool).getProjectByProjectInput({ id }) @@ -759,35 +744,27 @@ export const getAllProjectsInGroup: ResolverFn = async ( logger.debug('No grant available for getAllProjectsInGroup'); return []; } else { - // get group from all keycloak groups apollo context - let group = []; - if (groupInput.name) { - group = await getGroupFromGroupsByName(groupInput.name, keycloakGroups); - } - if (groupInput.id) { - group = await getGroupFromGroupsById(groupInput.id, keycloakGroups); - } - // get users groups from users keycloak groups apollo context - const userGroups = keycloakUsersGroups; + const group = await models.GroupModel.loadGroupByIdOrName(groupInput); + const userGroups = await models.UserModel.getAllGroupsForUser(keycloakGrant.access_token.content.sub); const user = await models.UserModel.loadUserById( keycloakGrant.access_token.content.sub ); const usersOrgs = R.defaultTo('', R.prop('lagoon-organizations', user.attributes)).toString() const usersOrgsViewer = R.defaultTo('', R.prop('lagoon-organizations-viewer', user.attributes)).toString() if (usersOrgs != "" ) { - const usersOrgsArr = usersOrgs.split(','); - for (const userOrg of usersOrgsArr) { - const orgGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(userOrg, keycloakGroups); + const uOrgs = usersOrgs.split(','); + for (const userOrg of uOrgs) { + const orgGroups = await Helpers(sqlClientPool).selectGroupsByOrganizationId(models, userOrg) for (const pGroup of orgGroups) { userGroups.push(pGroup) } } } if (usersOrgsViewer != "" ) { - const usersOrgsArr = usersOrgsViewer.split(','); - for (const userOrg of usersOrgsArr) { - const orgViewerGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(userOrg, keycloakGroups); - for (const pGroup of orgViewerGroups) { + const uOrgs = usersOrgsViewer.split(','); + for (const userOrg of uOrgs) { + const orgGroups = await Helpers(sqlClientPool).selectGroupsByOrganizationId(models, userOrg) + for (const pGroup of orgGroups) { userGroups.push(pGroup) } } diff --git a/services/api/src/resources/group/sql.ts b/services/api/src/resources/group/sql.ts new file mode 100644 index 0000000000..47f254cb6a --- /dev/null +++ b/services/api/src/resources/group/sql.ts @@ -0,0 +1,76 @@ +import { knex } from '../../util/db'; + +import { logger } from '../../loggers/logger'; + +export const Sql = { + selectProjectIdsByGroupIDs: (groupIds: string[]) => + knex('kc_group_projects') + .whereIn('group_id', groupIds) + .select(knex.raw('group_concat(project_id) as project_ids')) + .toString(), + selectProjectIdsByGroupID: (groupId: string) => + knex('kc_group_projects') + .where('group_id', groupId) + .select(knex.raw('group_concat(project_id) as project_ids')) + .toString(), + selectGroupsByProjectId: (projectId: number) => + knex('kc_group_projects') + .where('project_id', projectId) + .select(knex.raw('group_concat(group_id) as group_ids')) + .toString(), + selectGroupsByOrganizationId: (organizationId: number) => + knex('kc_group_organization') + .where('organization_id', organizationId) + .select(knex.raw('group_concat(group_id) as group_ids')) + .toString(), + selectOrganizationByGroupId: (groupId: string) => + knex('kc_group_organization') + .where('group_id', groupId) + .toString(), + addProjectToGroup: (input) => { + const { + id, + projectId, + groupId, + } = input; + return knex('kc_group_projects').insert({ + id, + projectId, + groupId, + }) + .onConflict('id') + .merge() + .toString(); + }, + removeProjectFromGroup: (projectId: number, groupId: string) => + knex('kc_group_projects') + .where('project_id', projectId) + .andWhere('group_id', groupId) + .del() + .toString(), + addOrganizationToGroup: (input) => { + const { + id, + groupId, + organizationId, + } = input; + return knex('kc_group_organization').insert({ + id, + groupId, + organizationId + }) + .onConflict('id') + .merge() + .toString(); + }, + deleteProjectGroup: (groupId: string) => + knex('kc_group_projects') + .where('group_id', groupId) + .del() + .toString(), + deleteOrganizationGroup: (groupId: string) => + knex('kc_group_organization') + .where('group_id', groupId) + .del() + .toString(), +}; \ No newline at end of file diff --git a/services/api/src/resources/index.ts b/services/api/src/resources/index.ts index 52c6368bd1..8d259322e3 100644 --- a/services/api/src/resources/index.ts +++ b/services/api/src/resources/index.ts @@ -20,7 +20,6 @@ export interface ResolverFn { ProjectModel, EnvironmentModel, }, - keycloakGroups?: any | null, keycloakUsersGroups?: any | null, adminScopes?: any | null, }, diff --git a/services/api/src/resources/organization/resolvers.ts b/services/api/src/resources/organization/resolvers.ts index 6cb0161ba9..b53a85c8fb 100644 --- a/services/api/src/resources/organization/resolvers.ts +++ b/services/api/src/resources/organization/resolvers.ts @@ -9,6 +9,7 @@ import { Sql } from './sql'; import { arrayDiff } from '../../util/func'; import { Helpers as openshiftHelpers } from '../openshift/helpers'; import { Helpers as notificationHelpers } from '../notification/helpers'; +import { Helpers as groupHelpers } from '../group/helpers'; import validator from 'validator'; import { log } from 'winston'; @@ -340,7 +341,7 @@ export const getNotificationsForOrganizationProjectId: ResolverFn = async ( export const getOwnersByOrganizationId: ResolverFn = async ( { id: oid }, _input, - { hasPermission, models, keycloakGrant, keycloakUsersGroups } + { hasPermission, models } ) => { await hasPermission('organization', 'view', { organization: oid, @@ -353,13 +354,13 @@ export const getOwnersByOrganizationId: ResolverFn = async ( export const getGroupsByOrganizationId: ResolverFn = async ( { id: oid }, _input, - { hasPermission, models, keycloakGrant, keycloakGroups } + { hasPermission, models, sqlClientPool } ) => { await hasPermission('organization', 'viewGroup', { organization: oid, }); - const orgGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(oid, keycloakGroups); + const orgGroups = await groupHelpers(sqlClientPool).selectGroupsByOrganizationId(models, oid) return orgGroups; }; @@ -368,13 +369,13 @@ export const getGroupsByOrganizationId: ResolverFn = async ( export const getUsersByOrganizationId: ResolverFn = async ( _, args, - { hasPermission, models, keycloakGrant, keycloakGroups } + { hasPermission, models, sqlClientPool } ) => { await hasPermission('organization', 'viewUsers', { organization: args.organization, }); - const orgGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(args.organization, keycloakGroups); + const orgGroups = await groupHelpers(sqlClientPool).selectGroupsByOrganizationId(models, args.organization) let members = [] for (const group in orgGroups) { @@ -450,7 +451,7 @@ export const getUserByEmailAndOrganizationId: ResolverFn = async ( export const getGroupRolesByUserIdAndOrganization: ResolverFn =async ( { id: uid, organization }, _input, - { hasPermission, models, keycloakGrant, keycloakUsersGroups, adminScopes } + { hasPermission, models, adminScopes } ) => { if (organization) { const queryUserGroups = await models.UserModel.getAllGroupsForUser(uid, organization); @@ -497,9 +498,9 @@ export const getGroupsByNameAndOrganizationId: ResolverFn = async ( export const getGroupCountByOrganizationProject: ResolverFn = async ( { id: pid }, _input, - { sqlClientPool, models, keycloakGroups } + { sqlClientPool, models } ) => { - const orgProjectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(pid, keycloakGroups); + const orgProjectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, pid) return orgProjectGroups.length } @@ -510,9 +511,9 @@ export const getGroupCountByOrganizationProject: ResolverFn = async ( export const getGroupsByOrganizationsProject: ResolverFn = async ( { id: pid }, _input, - { sqlClientPool, models, keycloakGrant, keycloakGroups, keycloakUsersGroups, adminScopes } + { sqlClientPool, models, keycloakGrant, keycloakUsersGroups, adminScopes } ) => { - const orgProjectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(pid, keycloakGroups); + const orgProjectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, pid) if (adminScopes.projectViewAll) { // if platform owner, this will show ALL groups on a project (those that aren't in the organization too, yes its possible with outside intervention :| ) return orgProjectGroups; @@ -532,7 +533,7 @@ export const getGroupsByOrganizationsProject: ResolverFn = async ( for (const userOrg of usersOrgsArr) { const project = await projectHelpers(sqlClientPool).getProjectById(pid); if (project.organization == userOrg) { - const orgGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(project.organization, keycloakGroups); + const orgGroups = await groupHelpers(sqlClientPool).selectGroupsByOrganizationId(models, project.organization) for (const pGroup of orgGroups) { userGroups.push(pGroup) } @@ -544,8 +545,8 @@ export const getGroupsByOrganizationsProject: ResolverFn = async ( for (const userOrg of usersOrgsArr) { const project = await projectHelpers(sqlClientPool).getProjectById(pid); if (project.organization == userOrg) { - const orgViewerGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(project.organization, keycloakGroups); - for (const pGroup of orgViewerGroups) { + const orgGroups = await groupHelpers(sqlClientPool).selectGroupsByOrganizationId(models, project.organization) + for (const pGroup of orgGroups) { userGroups.push(pGroup) } } @@ -623,7 +624,7 @@ const checkProjectGroupAssociation = async (oid, projectGroups, projectGroupName export const getProjectGroupOrganizationAssociation: ResolverFn = async ( _root, { input }, - { sqlClientPool, models, hasPermission, keycloakGroups } + { sqlClientPool, models, hasPermission } ) => { let pid = input.project; let oid = input.organization; @@ -637,7 +638,7 @@ export const getProjectGroupOrganizationAssociation: ResolverFn = async ( const otherOrgs = [] // get all the groups the requested project is in - const projectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(pid, keycloakGroups); + const projectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, pid) await checkProjectGroupAssociation(oid, projectGroups, projectGroupNames, otherOrgs, groupProjectIds, projectInOtherOrgs, sqlClientPool) return "success"; @@ -649,7 +650,7 @@ export const getProjectGroupOrganizationAssociation: ResolverFn = async ( export const removeProjectFromOrganization: ResolverFn = async ( root, { input }, - { sqlClientPool, hasPermission, models, keycloakGroups, userActivityLogger } + { sqlClientPool, hasPermission, models, userActivityLogger } ) => { // platform admin only await hasPermission('organization', 'add'); @@ -663,7 +664,7 @@ export const removeProjectFromOrganization: ResolverFn = async ( } try { - const projectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(pid, keycloakGroups); + const projectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, pid) let removeGroups = [] for (const g in projectGroups) { @@ -735,7 +736,7 @@ export const removeProjectFromOrganization: ResolverFn = async ( export const addExistingProjectToOrganization: ResolverFn = async ( root, { input }, - { sqlClientPool, hasPermission, userActivityLogger, models, keycloakGroups } + { sqlClientPool, hasPermission, userActivityLogger, models } ) => { let pid = input.project; @@ -750,7 +751,7 @@ export const addExistingProjectToOrganization: ResolverFn = async ( const otherOrgs = [] // get all the groups the requested project is in - const projectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(pid, keycloakGroups); + const projectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, pid) await checkProjectGroupAssociation(oid, projectGroups, projectGroupNames, otherOrgs, groupProjectIds, projectInOtherOrgs, sqlClientPool) // check if project.organization is already set? @@ -923,7 +924,7 @@ export const addExistingGroupToOrganization: ResolverFn = async ( export const removeUserFromOrganizationGroups: ResolverFn = async ( _root, { input: { user: userInput, organization: organizationInput } }, - { models, sqlClientPool, hasPermission, keycloakGroups, userActivityLogger } + { models, sqlClientPool, hasPermission, userActivityLogger } ) => { if (R.isEmpty(userInput)) { @@ -945,7 +946,7 @@ export const removeUserFromOrganizationGroups: ResolverFn = async ( await hasPermission('organization', 'removeGroup', { organization: organizationInput, }); - const orgGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(organizationInput, keycloakGroups); + const orgGroups = await groupHelpers(sqlClientPool).selectGroupsByOrganizationId(models, organizationInput) // iterate through groups and remove the user let groupsRemoved = [] @@ -981,7 +982,7 @@ export const removeUserFromOrganizationGroups: ResolverFn = async ( export const deleteOrganization: ResolverFn = async ( _root, { input }, - { sqlClientPool, hasPermission, userActivityLogger, models, keycloakGroups } + { sqlClientPool, hasPermission, userActivityLogger, models } ) => { await hasPermission('organization', 'delete', { organization: input.id @@ -1014,7 +1015,7 @@ export const deleteOrganization: ResolverFn = async ( ); } - const orgGroups = await models.GroupModel.loadGroupsByOrganizationIdFromGroups(orgResult.id, keycloakGroups); + const orgGroups = await groupHelpers(sqlClientPool).selectGroupsByOrganizationId(models, orgResult.id) if (orgGroups.length > 0) { // throw error if there are any existing environments throw new Error( @@ -1050,9 +1051,9 @@ export const deleteOrganization: ResolverFn = async ( return 'success'; }; -const checkBulkProjectGroupAssociation = async (oid, pid, projectsToMove, groupsToMove, projectsInOtherOrgs, groupsInOtherOrgs, sqlClientPool, models, keycloakGroups) => { +const checkBulkProjectGroupAssociation = async (oid, pid, projectsToMove, groupsToMove, projectsInOtherOrgs, groupsInOtherOrgs, sqlClientPool, models) => { const groupProjectIds = []; - const projectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(pid, keycloakGroups); + const projectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, pid) // get all the groups the requested project is in for (const group of projectGroups) { // for each group the project is in, get the list of projects that are also in this group @@ -1069,7 +1070,7 @@ const checkBulkProjectGroupAssociation = async (oid, pid, projectsToMove, groups if (groupProjectIds.length > 0) { for (const pGroup of groupProjectIds) { const project = await projectHelpers(sqlClientPool).getProjectById(pGroup.project) - const projectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(project.id, keycloakGroups); + const projectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, pid) // check if the project is already in the requested organization if (project.organization != oid && project.organization == null) { let alreadyAdded = false @@ -1133,7 +1134,7 @@ const checkBulkProjectGroupAssociation = async (oid, pid, projectsToMove, groups export const checkBulkImportProjectsAndGroupsToOrganization: ResolverFn = async ( _root, { input }, - { sqlClientPool, models, hasPermission, keycloakGroups } + { sqlClientPool, models, hasPermission } ) => { let pid = input.project; let oid = input.organization; @@ -1147,7 +1148,7 @@ export const checkBulkImportProjectsAndGroupsToOrganization: ResolverFn = async const groupsInOtherOrgs = [] // get all the groups the requested project is in - await checkBulkProjectGroupAssociation(oid, pid, projectsToMove, groupsToMove, projectsInOtherOrgs, groupsInOtherOrgs, sqlClientPool, models, keycloakGroups) + await checkBulkProjectGroupAssociation(oid, pid, projectsToMove, groupsToMove, projectsInOtherOrgs, groupsInOtherOrgs, sqlClientPool, models) return { projects: projectsToMove, groups: groupsToMove, otherOrgProjects: projectsInOtherOrgs, otherOrgGroups: groupsInOtherOrgs }; }; @@ -1157,7 +1158,7 @@ export const checkBulkImportProjectsAndGroupsToOrganization: ResolverFn = async export const bulkImportProjectsAndGroupsToOrganization: ResolverFn = async ( root, { input, detachNotifications }, - { sqlClientPool, hasPermission, userActivityLogger, models, keycloakGroups } + { sqlClientPool, hasPermission, userActivityLogger, models } ) => { let pid = input.project; @@ -1172,7 +1173,7 @@ export const bulkImportProjectsAndGroupsToOrganization: ResolverFn = async ( const groupsInOtherOrgs = [] // get all the groups the requested project is in - await checkBulkProjectGroupAssociation(oid, pid, projectsToMove, groupsToMove, projectsInOtherOrgs, groupsInOtherOrgs, sqlClientPool, models, keycloakGroups) + await checkBulkProjectGroupAssociation(oid, pid, projectsToMove, groupsToMove, projectsInOtherOrgs, groupsInOtherOrgs, sqlClientPool, models) // if anything comes back in projectsInOtherOrgs or groupsInOtherOrgs, then this mutation should fail and inform the user // to run the query first and return the fields that contain information about why it can't move the projects @@ -1197,6 +1198,9 @@ export const bulkImportProjectsAndGroupsToOrganization: ResolverFn = async ( "lagoon-organization": [input.organization] } }); + if (input.organization) { + await groupHelpers(sqlClientPool).addOrganizationToGroup(input.organization, group.id) + } groupsDone.push(group.id) // log this activity diff --git a/services/api/src/resources/project/resolvers.ts b/services/api/src/resources/project/resolvers.ts index d2af5630e7..247f184838 100644 --- a/services/api/src/resources/project/resolvers.ts +++ b/services/api/src/resources/project/resolvers.ts @@ -18,6 +18,7 @@ import { Sql as sshKeySql } from '../sshKey/sql'; import { createHarborOperations } from './harborSetup'; import { Helpers as organizationHelpers } from '../organization/helpers'; import { Helpers as notificationHelpers } from '../notification/helpers'; +import { Helpers as groupHelpers } from '../group/helpers'; import { getUserProjectIdsFromRoleProjectIds } from '../../util/auth'; import GitUrlParse from 'git-url-parse'; @@ -82,10 +83,7 @@ export const getAllProjects: ResolverFn = async ( return []; } // get the project ids from the users groups - const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser({ - id: keycloakGrant.access_token.content.sub, - - }, keycloakUsersGroups); + const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser(keycloakGrant.access_token.content.sub, keycloakUsersGroups); userProjectIds = getUserProjectIdsFromRoleProjectIds(userProjectRoles); } @@ -201,9 +199,7 @@ export const getProjectsByMetadata: ResolverFn = async ( return []; } - const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser({ - id: keycloakGrant.access_token.content.sub - }, keycloakUsersGroups); + const userProjectRoles = await models.UserModel.getAllProjectsIdsForUser(keycloakGrant.access_token.content.sub, keycloakUsersGroups); userProjectIds = getUserProjectIdsFromRoleProjectIds(userProjectRoles); } @@ -400,7 +396,7 @@ export const addProject = async ( group = await models.GroupModel.addGroup({ name: `project-${project.name}`, attributes: attributes - }); + }, project.id, input.organization); } catch (err) { logger.error( `Could not create default project group for ${project.name}: ${err.message}` @@ -502,7 +498,7 @@ export const addProject = async ( export const deleteProject: ResolverFn = async ( _root, { input: { project: projectName } }, - { sqlClientPool, hasPermission, userActivityLogger, models, keycloakGroups } + { sqlClientPool, hasPermission, userActivityLogger, models } ) => { // Will throw on invalid conditions const pid = await Helpers(sqlClientPool).getProjectIdByName(projectName); @@ -545,7 +541,7 @@ export const deleteProject: ResolverFn = async ( // Remove the project from all groups it is associated to try { - const projectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups(pid, keycloakGroups); + const projectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, pid) // @TODO: use the new helper instead in the following for loop, once the `opendistrosecurityoperations` stuff goes away // await models.GroupModel.removeProjectFromGroups(pid, projectGroups); for (const groupInput of projectGroups) { diff --git a/services/api/src/resources/task/task_definition_resolvers.ts b/services/api/src/resources/task/task_definition_resolvers.ts index 50b2a4129d..664fdda8e6 100644 --- a/services/api/src/resources/task/task_definition_resolvers.ts +++ b/services/api/src/resources/task/task_definition_resolvers.ts @@ -5,6 +5,7 @@ import { Helpers } from './helpers'; import { Filters } from './filters'; import { Helpers as environmentHelpers } from '../environment/helpers'; import { Helpers as projectHelpers } from '../project/helpers'; +import { Helpers as groupHelpers } from '../group/helpers'; import { Helpers as deploymentHelpers } from '../deployment/helpers'; import { Validators as envValidators } from '../environment/validators'; import { @@ -22,6 +23,7 @@ import { IKeycloakAuthAttributes, KeycloakUnauthorizedError } from '../../util/a import { Environment } from '../../resolvers'; import { generateTaskName } from '@lagoon/commons/dist/util/lagoon'; import { logger } from '../../loggers/logger'; +import sql from '../workflow/sql'; enum AdvancedTaskDefinitionTarget { Group, @@ -88,7 +90,7 @@ export const advancedTaskDefinitionById = async ( export const getRegisteredTasksByEnvironmentId = async ( { id }, {}, - { sqlClientPool, hasPermission, models, keycloakGroups } + { sqlClientPool, hasPermission, models } ) => { let rows; @@ -96,7 +98,7 @@ export const getRegisteredTasksByEnvironmentId = async ( rows = await resolveTasksForEnvironment( {}, { environment: id }, - { sqlClientPool, hasPermission, models, keycloakGroups } + { sqlClientPool, hasPermission, models } ); rows = await Filters.filterAdminTasks(hasPermission, rows); @@ -108,7 +110,7 @@ export const getRegisteredTasksByEnvironmentId = async ( export const resolveTasksForEnvironment = async ( root, { environment }, - { sqlClientPool, hasPermission, models, keycloakGroups } + { sqlClientPool, hasPermission, models } ) => { const environmentDetails = await environmentHelpers( sqlClientPool @@ -131,10 +133,7 @@ export const resolveTasksForEnvironment = async ( Sql.selectAdvancedTaskDefinitionsForProject(proj.project) ); - const projectGroups = await models.GroupModel.loadGroupsByProjectIdFromGroups( - proj.projectId, - keycloakGroups - ); + const projectGroups = await groupHelpers(sqlClientPool).selectGroupsByProjectId(models, proj.projectId) const projectGroupsFiltered = R.pluck('name', projectGroups); @@ -531,7 +530,7 @@ const getProjectByEnvironmentIdOrProjectId = async ( export const invokeRegisteredTask = async ( root, { advancedTaskDefinition, environment, argumentValues, sourceType }, - { sqlClientPool, hasPermission, models, keycloakGroups, keycloakGrant, legacyGrant } + { sqlClientPool, hasPermission, models, keycloakGrant, legacyGrant } ) => { await envValidators(sqlClientPool).environmentExists(environment); @@ -540,8 +539,7 @@ export const invokeRegisteredTask = async ( hasPermission, advancedTaskDefinition, environment, - models, - keycloakGroups + models ); const atb = advancedTaskToolbox.advancedTaskFunctions( @@ -666,15 +664,14 @@ const getNamedAdvancedTaskForEnvironment = async ( hasPermission, advancedTaskDefinition, environment, - models, - keycloakGroups + models ):Promise => { let rows; rows = await resolveTasksForEnvironment( {}, { environment }, - { sqlClientPool, hasPermission, models, keycloakGroups } + { sqlClientPool, hasPermission, models } ); rows = await Filters.filterAdminTasks(hasPermission, rows); diff --git a/services/api/src/util/auth.ts b/services/api/src/util/auth.ts index 666c84e119..4bc3cc5c7a 100644 --- a/services/api/src/util/auth.ts +++ b/services/api/src/util/auth.ts @@ -244,7 +244,7 @@ export const keycloakHasPermission = (grant, requestCache, modelClients, service // but could happen elsewhere const keycloakUsersGroups = await UserModel.getAllGroupsForUser(currentUser.id); // grab the users project ids and roles in the first request - groupRoleProjectIds = await UserModel.getAllProjectsIdsForUser(currentUser, keycloakUsersGroups); + groupRoleProjectIds = await UserModel.getAllProjectsIdsForUser(currentUser.id, keycloakUsersGroups); [highestRoleForProject, upids] = getUserRoleForProjectFromRoleProjectIds(groupRoleProjectIds, projectId) } diff --git a/services/api/src/util/db.ts b/services/api/src/util/db.ts index c5295e9b93..37f2b8990b 100644 --- a/services/api/src/util/db.ts +++ b/services/api/src/util/db.ts @@ -21,6 +21,26 @@ export const knex = require('knex')({ } }); +// set up the lagoon migrations table +export const migrate = require('knex')({ + client: 'mysql', + connection: { + host : process.env.API_DB_HOST || 'api-db', + port : process.env.API_DB_PORT || 3306, + user : process.env.API_DB_USER || 'api', + password : process.env.API_DB_PASSWORD || 'api', + database : process.env.API_DB_DATABASE || 'infrastructure' + }, + pool: { + min: 2, + max: 10 + }, + migrations: { + tableName: 'lagoon_migrations', + directory: 'dist/migrations/lagoon/migrations' + }, +}); + /** * "Temporary" query utility for mariadb to replace mariasql. Intended to * reduce the pain of switching, not to markedly improve anything. diff --git a/services/keycloak/Dockerfile b/services/keycloak/Dockerfile index b40a2c6043..5e40050355 100644 --- a/services/keycloak/Dockerfile +++ b/services/keycloak/Dockerfile @@ -1,3 +1,8 @@ +FROM maven:3.8.2-jdk-11 as builder +# build the custom token mapper in builder +COPY custom-mapper/. . +RUN mvn clean compile package + FROM registry.access.redhat.com/ubi9 AS ubi-micro-build RUN mkdir -p /mnt/rootfs RUN dnf install --installroot /mnt/rootfs nc jq openssl curl unzip --releasever 9 --setopt install_weak_deps=false --nodocs -y; dnf --installroot /mnt/rootfs clean all @@ -64,6 +69,11 @@ ENV TMPDIR=/tmp \ KEYCLOAK_API_CLIENT_SECRET=39d5282d-3684-4026-b4ed-04bbc034b61a \ KEYCLOAK_AUTH_SERVER_CLIENT_SECRET=f605b150-7636-4447-abd3-70988786b330 \ KEYCLOAK_SERVICE_API_CLIENT_SECRET=d3724d52-34d1-4967-a802-4d178678564b \ + LAGOON_DB_VENDOR=mariadb \ + LAGOON_DB_DATABASE=infrastructure \ + LAGOON_DB_USER=api \ + LAGOON_DB_PASSWORD=api \ + LAGOON_DB_HOST=api-db \ JAVA_OPTS="-server -Xms2048m -Xmx4096m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true" VOLUME /opt/keycloak/data @@ -71,9 +81,11 @@ VOLUME /opt/keycloak/data COPY entrypoints/kc-startup.sh /lagoon/kc-startup.sh COPY entrypoints/wait-for-mariadb.sh /lagoon/entrypoints/98-wait-for-mariadb.sh COPY entrypoints/default-keycloak-entrypoint.sh /lagoon/entrypoints/99-default-keycloak-entrypoint.sh + COPY startup-scripts /opt/keycloak/startup-scripts COPY themes/lagoon /opt/keycloak/themes/lagoon COPY --from=commons /tmp/lagoon-scripts.jar /opt/keycloak/providers/lagoon-scripts.jar +COPY --from=builder /target/custom-protocol-mapper-1.0.0.jar /opt/keycloak/providers/custom-protocol-mapper-1.0.0.jar COPY lagoon-realm-2.16.0.json /lagoon/seed/lagoon-realm-2.16.0.json diff --git a/services/keycloak/custom-mapper/pom.xml b/services/keycloak/custom-mapper/pom.xml new file mode 100644 index 0000000000..2b6bcec1f3 --- /dev/null +++ b/services/keycloak/custom-mapper/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + + net.cake.keycloak.custom + custom-protocol-mapper + 1.0.0 + jar + + + 17.0.1 + + + + + + org.keycloak + keycloak-core + ${keycloak.version} + provided + + + org.keycloak + keycloak-server-spi + ${keycloak.version} + provided + + + org.keycloak + keycloak-server-spi-private + ${keycloak.version} + provided + + + org.keycloak + keycloak-services + ${keycloak.version} + provided + + + org.mariadb.jdbc + mariadb-java-client + LATEST + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + true + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.0 + + + + package + + shade + + + + + + + \ No newline at end of file diff --git a/services/keycloak/custom-mapper/src/main/java/CustomOIDCProtocolMapper.java b/services/keycloak/custom-mapper/src/main/java/CustomOIDCProtocolMapper.java new file mode 100644 index 0000000000..16d9edd39e --- /dev/null +++ b/services/keycloak/custom-mapper/src/main/java/CustomOIDCProtocolMapper.java @@ -0,0 +1,169 @@ +import org.keycloak.models.ClientSessionContext; +import org.keycloak.models.KeycloakSession; +import org.keycloak.models.ProtocolMapperModel; +import org.keycloak.models.UserSessionModel; +import org.keycloak.protocol.oidc.OIDCLoginProtocol; +import org.keycloak.protocol.oidc.mappers.*; +import org.keycloak.provider.ProviderConfigProperty; +import org.keycloak.representations.AccessToken; +import org.jboss.logging.Logger; + +import org.keycloak.models.IdentityProviderMapperModel; +import org.keycloak.models.RealmModel; +import org.keycloak.models.GroupModel; +import org.keycloak.models.UserModel; +import org.keycloak.models.RoleModel; +import org.keycloak.models.utils.KeycloakModelUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Iterator; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Statement; +import java.sql.PreparedStatement; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class CustomOIDCProtocolMapper extends AbstractOIDCProtocolMapper implements OIDCAccessTokenMapper, OIDCIDTokenMapper, UserInfoTokenMapper { + + public static final String PROVIDER_ID = "lagoon-search-customprotocolmapper"; + + private static final Logger logger = Logger.getLogger(CustomOIDCProtocolMapper.class); + + private static final List configProperties = new ArrayList(); + + /** + * Maybe you want to have config fields for your Mapper + */ + /* + static { + ProviderConfigProperty property; + property = new ProviderConfigProperty(); + property.setName(ProtocolMapperUtils.USER_ATTRIBUTE); + property.setLabel(ProtocolMapperUtils.USER_MODEL_ATTRIBUTE_LABEL); + property.setHelpText(ProtocolMapperUtils.USER_MODEL_ATTRIBUTE_HELP_TEXT); + property.setType(ProviderConfigProperty.STRING_TYPE); + configProperties.add(property); + + property = new ProviderConfigProperty(); + property.setName(ProtocolMapperUtils.MULTIVALUED); + property.setLabel(ProtocolMapperUtils.MULTIVALUED_LABEL); + property.setHelpText(ProtocolMapperUtils.MULTIVALUED_HELP_TEXT); + property.setType(ProviderConfigProperty.BOOLEAN_TYPE); + configProperties.add(property); + + } + */ + @Override + public List getConfigProperties() { + return configProperties; + } + + @Override + public String getDisplayCategory() { + return TOKEN_MAPPER_CATEGORY; + } + + @Override + public String getDisplayType() { + return "Lagoon Project Group Mapper"; + } + + @Override + public String getId() { + return PROVIDER_ID; + } + + @Override + public String getHelpText() { + return "A mapper that can retrieve groups and projects from the lagoon API to store in the token"; + } + + public AccessToken transformAccessToken(AccessToken token, ProtocolMapperModel mappingModel, KeycloakSession keycloakSession, + UserSessionModel userSession, ClientSessionContext clientSessionCtx) { + + List groupsAndRoles = new ArrayList(); + Map groupProjectIds = new HashMap(); + Map projectGroupProjectIds = new HashMap(); + UserModel user = userSession.getUser(); + Set groups = user.getGroupsStream().collect(Collectors.toSet());; + for (GroupModel group : groups) { + if(group.getFirstAttribute("type").equals("role-subgroup")) { + GroupModel parent = group.getParent(); + String parentName = parent.getName(); + List projectIds = new ArrayList(); + try (Connection c = DbUtil.getConnection()) { + PreparedStatement statement = c.prepareStatement("SELECT project_id FROM kc_group_projects WHERE group_id='"+parent.getId()+"'"); + ResultSet resultSet = statement.executeQuery(); + while (resultSet.next()) { + projectIds.add(resultSet.getString(1)); + } + } + catch(Exception ex) { + // don't throw an exception, just log and continue so that the token still generates + logger.tracef("Issue connecting to database to perform query on group id %s: %v", parent.getId(), ex); + } + String temp = parent.getFirstAttribute("type"); + String result = ""; + if(temp != null && !temp.isEmpty()){ + result = temp; + } + if(result.equals("project-default-group")) { + if(projectIds != null) { + for (String projectId : projectIds) { + projectGroupProjectIds.put(projectId, parentName); + }; + } + } else { + if(projectIds != null) { + // add the group so group-tenant association works properly + groupsAndRoles.add(parentName); + // calculate the groupprojectids + for (String projectId : projectIds) { + groupProjectIds.put(projectId, parentName); + }; + } + } + } + }; + // that remains are project ids that are not already associated to an existing group + projectGroupProjectIds.keySet().removeAll(groupProjectIds.keySet()); + + for (Object projectId : projectGroupProjectIds.keySet()) { + groupsAndRoles.add("p"+projectId); + } + + // add all roles the user is part of + Set userRoles = user.getRoleMappingsStream().collect(Collectors.toSet()); + for (RoleModel role : userRoles) { + String roleName = role.getName(); + groupsAndRoles.add(roleName); + }; + + token.getOtherClaims().put("groups", groupsAndRoles); + + setClaim(token, mappingModel, userSession, keycloakSession, clientSessionCtx); + return token; + } + + public static ProtocolMapperModel create(String name, + boolean accessToken, boolean idToken, boolean userInfo) { + ProtocolMapperModel mapper = new ProtocolMapperModel(); + mapper.setName(name); + mapper.setProtocolMapper(PROVIDER_ID); + mapper.setProtocol(OIDCLoginProtocol.LOGIN_PROTOCOL); + Map config = new HashMap(); + config.put(OIDCAttributeMapperHelper.INCLUDE_IN_ACCESS_TOKEN, "true"); + config.put(OIDCAttributeMapperHelper.INCLUDE_IN_ID_TOKEN, "true"); + config.put(OIDCAttributeMapperHelper.INCLUDE_IN_USERINFO, "true"); + mapper.setConfig(config); + return mapper; + } + + +} diff --git a/services/keycloak/custom-mapper/src/main/java/DbUtil.java b/services/keycloak/custom-mapper/src/main/java/DbUtil.java new file mode 100644 index 0000000000..0a54d3edd6 --- /dev/null +++ b/services/keycloak/custom-mapper/src/main/java/DbUtil.java @@ -0,0 +1,20 @@ +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import org.mariadb.jdbc.Driver; + +import org.keycloak.component.ComponentModel; + +public class DbUtil { + + public static Connection getConnection() throws SQLException{ + String driverClass = System.getenv("LAGOON_DB_VENDOR"); + String username = System.getenv("LAGOON_DB_USER"); + String password = System.getenv("LAGOON_DB_PASSWORD"); + String database = System.getenv("LAGOON_DB_DATABASE"); + String host = System.getenv("LAGOON_DB_HOST"); + + String jdbcUrl = "jdbc:"+driverClass+"://"+host+":3306/"+database+"?user="+username+"&password="+password; + return DriverManager.getConnection(jdbcUrl); + } +} \ No newline at end of file diff --git a/services/keycloak/custom-mapper/src/main/resources/META-INF/jboss-deployment-structure.xml b/services/keycloak/custom-mapper/src/main/resources/META-INF/jboss-deployment-structure.xml new file mode 100644 index 0000000000..47e418d700 --- /dev/null +++ b/services/keycloak/custom-mapper/src/main/resources/META-INF/jboss-deployment-structure.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/services/keycloak/custom-mapper/src/main/resources/META-INF/services/org.keycloak.protocol.ProtocolMapper b/services/keycloak/custom-mapper/src/main/resources/META-INF/services/org.keycloak.protocol.ProtocolMapper new file mode 100644 index 0000000000..c3f5a9dd43 --- /dev/null +++ b/services/keycloak/custom-mapper/src/main/resources/META-INF/services/org.keycloak.protocol.ProtocolMapper @@ -0,0 +1 @@ +CustomOIDCProtocolMapper \ No newline at end of file diff --git a/services/keycloak/startup-scripts/00-configure-lagoon.sh b/services/keycloak/startup-scripts/00-configure-lagoon.sh index 5541bd8165..65feb34349 100755 --- a/services/keycloak/startup-scripts/00-configure-lagoon.sh +++ b/services/keycloak/startup-scripts/00-configure-lagoon.sh @@ -138,8 +138,8 @@ function configure_lagoon_redirect_uris { update_redirect_uri+="\"$addr\"," done update_redirect_uri=$(echo $update_redirect_uri | sed 's/,*$//g')] - LAGOON_UI_CLIENT_ID=$(/opt/jboss/keycloak/bin/kcadm.sh get -r lagoon clients?clientId=lagoon-ui --config $CONFIG_PATH | jq -r '.[0]["id"]') - /opt/jboss/keycloak/bin/kcadm.sh update clients/${LAGOON_UI_CLIENT_ID} -s redirectUris=$update_redirect_uri --config "$CONFIG_PATH" -r ${KEYCLOAK_REALM:-master} + LAGOON_UI_CLIENT_ID=$( /opt/keycloak/bin/kcadm.sh get -r lagoon clients?clientId=lagoon-ui --config $CONFIG_PATH | jq -r '.[0]["id"]') + /opt/keycloak/bin/kcadm.sh update clients/${LAGOON_UI_CLIENT_ID} -s redirectUris=$update_redirect_uri --config "$CONFIG_PATH" -r ${KEYCLOAK_REALM:-master} fi } @@ -183,6 +183,27 @@ function check_migrations_version { EOF } +function migrate_to_custom_group_mapper { + local opendistro_security_client_id=$( /opt/keycloak/bin/kcadm.sh get -r lagoon clients?clientId=lagoon-opendistro-security --config $CONFIG_PATH | jq -r '.[0]["id"]') + local lagoon_opendistro_security_mappers=$( /opt/keycloak/bin/kcadm.sh get -r lagoon clients/$opendistro_security_client_id/protocol-mappers/models --config $CONFIG_PATH) + local lagoon_opendistro_security_mapper_groups=$(echo $lagoon_opendistro_security_mappers | jq -r '.[] | select(.name=="groups") | .protocolMapper') + if [ "$lagoon_opendistro_security_mapper_groups" == "lagoon-search-customprotocolmapper" ]; then + echo "custom mapper already migrated" + return 0 + fi + + echo Migrating "token mapper for search" to custom token mapper + + ################ + # Update Mapper + ################ + + local old_mapper_id=$(echo $lagoon_opendistro_security_mappers | jq -r '.[] | select(.name=="groups") | .id') + /opt/keycloak/bin/kcadm.sh delete -r lagoon clients/$opendistro_security_client_id/protocol-mappers/models/$old_mapper_id --config $CONFIG_PATH + echo '{"name":"groups","protocolMapper":"lagoon-search-customprotocolmapper","protocol":"openid-connect","config":{"id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","multivalued":"true","claim.name":"groups","jsonType.label":"String"}}' | /opt/keycloak/bin/kcadm.sh create -r ${KEYCLOAK_REALM:-master} clients/$opendistro_security_client_id/protocol-mappers/models --config $CONFIG_PATH -f - + +} + ################## # Initialization # ################## @@ -208,6 +229,7 @@ function configure_keycloak { configure_lagoon_redirect_uris check_migrations_version + migrate_to_custom_group_mapper #post 2.18.0+ migrations after this point # always run last diff --git a/yarn.lock b/yarn.lock index 99f0a8db82..8e0830e994 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2907,10 +2907,10 @@ color@^3.1.3: color-convert "^1.9.3" color-string "^1.6.0" -colorette@2.0.16: - version "2.0.16" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.16.tgz#713b9af84fdb000139f04546bd4a93f62a5085da" - integrity sha512-hUewv7oMjCp+wkBv5Rm0v87eJhq4woh5rSR+42YSQJKecCqgIqNkZ6lAlQms/BwHPJA5NKMRlpxPRv0n8HQW6g== +colorette@2.0.19: + version "2.0.19" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" + integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ== colors@1.4.0, colors@^1.1.2: version "1.4.0" @@ -2942,16 +2942,16 @@ commander@5.0.0: resolved "https://registry.yarnpkg.com/commander/-/commander-5.0.0.tgz#dbf1909b49e5044f8fdaf0adc809f0c0722bdfd0" integrity sha512-JrDGPAKjMGSP1G0DUoaceEJ3DZgAfr/q6X7FVk4+U5KxUSKviYGM2k6zWkfyyBHy5rAtzgYJFa1ro2O9PtoxwQ== +commander@^10.0.0: + version "10.0.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06" + integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== + commander@^2.20.3: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== -commander@^7.1.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" - integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== - common-tags@^1.4.0: version "1.8.2" resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.2.tgz#94ebb3c076d26032745fd54face7f688ef5ac9c6" @@ -3177,20 +3177,13 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@~2.6.9: dependencies: ms "2.0.0" -debug@4, debug@^4.1.0, debug@^4.1.1: +debug@4, debug@4.3.4, debug@^4.1.0, debug@^4.1.1: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" -debug@4.3.2: - version "4.3.2" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" - integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== - dependencies: - ms "2.1.2" - debug@=3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" @@ -4422,10 +4415,10 @@ get-value@^2.0.3, get-value@^2.0.6: resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" integrity sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA== -getopts@2.2.5: - version "2.2.5" - resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.5.tgz#67a0fe471cacb9c687d817cab6450b96dde8313b" - integrity sha512-9jb7AW5p3in+IiJWhQiZmmwkpLaR/ccTWdWQCtZM66HJcHHLegowh4q4tSD7gouUyeNvFWRavfK9GXosQHDpFA== +getopts@2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.3.0.tgz#71e5593284807e03e2427449d4f6712a268666f4" + integrity sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA== getpass@^0.1.1: version "0.1.7" @@ -6554,23 +6547,24 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -knex@^0.95.15: - version "0.95.15" - resolved "https://registry.yarnpkg.com/knex/-/knex-0.95.15.tgz#39d7e7110a6e2ad7de5d673d2dea94143015e0e7" - integrity sha512-Loq6WgHaWlmL2bfZGWPsy4l8xw4pOE+tmLGkPG0auBppxpI0UcK+GYCycJcqz9W54f2LiGewkCVLBm3Wq4ur/w== +knex@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/knex/-/knex-3.0.1.tgz#b12f3173c30d8c7b6d69dc257cc9c84db00ad60e" + integrity sha512-ruASxC6xPyDklRdrcDy6a9iqK+R9cGK214aiQa+D9gX2ZnHZKv6o6JC9ZfgxILxVAul4bZ13c3tgOAHSuQ7/9g== dependencies: - colorette "2.0.16" - commander "^7.1.0" - debug "4.3.2" + colorette "2.0.19" + commander "^10.0.0" + debug "4.3.4" escalade "^3.1.1" esm "^3.2.25" - getopts "2.2.5" + get-package-type "^0.1.0" + getopts "2.3.0" interpret "^2.2.0" lodash "^4.17.21" - pg-connection-string "2.5.0" - rechoir "0.7.0" + pg-connection-string "2.6.1" + rechoir "^0.8.0" resolve-from "^5.0.0" - tarn "^3.0.1" + tarn "^3.0.2" tildify "2.0.0" kubernetes-client@^3.15.0: @@ -7789,10 +7783,10 @@ performance-now@2.1.0, performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== -pg-connection-string@2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" - integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== +pg-connection-string@2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.1.tgz#78c23c21a35dd116f48e12e23c0965e8d9e2cbfb" + integrity sha512-w6ZzNu6oMmIzEAYVw+RLK0+nqHPt8K3ZnknKi+g48Ak2pr3dtljJW3o+D/n2zzCG07Zoe9VOX3aiKpj+BN0pjg== picocolors@^0.2.1: version "0.2.1" @@ -8414,12 +8408,12 @@ realpath-native@^1.1.0: dependencies: util.promisify "^1.0.0" -rechoir@0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.7.0.tgz#32650fd52c21ab252aa5d65b19310441c7e03aca" - integrity sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q== +rechoir@^0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.8.0.tgz#49f866e0d32146142da3ad8f0eff352b3215ff22" + integrity sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ== dependencies: - resolve "^1.9.0" + resolve "^1.20.0" redis-commands@^1.7.0: version "1.7.0" @@ -8609,7 +8603,7 @@ resolve@1.1.7: resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" integrity sha512-9znBF0vBcaSN3W2j7wKvdERPwqTxSpCq+if5C0WoTCyV9n24rua28jeuQ2pL/HOf+yUe/Mef+H/5p60K0Id3bg== -resolve@1.x, resolve@^1.10.0, resolve@^1.18.1, resolve@^1.22.4, resolve@^1.9.0: +resolve@1.x, resolve@^1.10.0, resolve@^1.18.1, resolve@^1.20.0, resolve@^1.22.4: version "1.22.8" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== @@ -9468,7 +9462,7 @@ table@4.0.2: slice-ansi "1.0.0" string-width "^2.1.1" -tarn@^3.0.1: +tarn@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/tarn/-/tarn-3.0.2.tgz#73b6140fbb881b71559c4f8bfde3d9a4b3d27693" integrity sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==