From ae1262c1024557078622b4395b32c69a8fbf8e93 Mon Sep 17 00:00:00 2001 From: Edwin Mackenzie-Owen Date: Mon, 4 Mar 2024 14:00:23 +0100 Subject: [PATCH] webhooks: only pull project for which request was received (#793) If a webhook request's project matches a wildcard, pull only that project instead of pulling the entire wildcard (which can be expensive for wildcards matching many projects). --- pkg/controller/controller.go | 1 + pkg/controller/projects.go | 31 +++++++++++++++++++++++++++++++ pkg/controller/scheduler.go | 7 +++++++ pkg/controller/webhooks.go | 16 ++++++++-------- pkg/schemas/tasks.go | 3 +++ 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 8aa123e0..7dcad65a 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -77,6 +77,7 @@ func (c *Controller) registerTasks() { schemas.TaskTypePullEnvironmentsFromProject: c.TaskHandlerPullEnvironmentsFromProject, schemas.TaskTypePullEnvironmentsFromProjects: c.TaskHandlerPullEnvironmentsFromProjects, schemas.TaskTypePullMetrics: c.TaskHandlerPullMetrics, + schemas.TaskTypePullProject: c.TaskHandlerPullProject, schemas.TaskTypePullProjectsFromWildcard: c.TaskHandlerPullProjectsFromWildcard, schemas.TaskTypePullProjectsFromWildcards: c.TaskHandlerPullProjectsFromWildcards, schemas.TaskTypePullRefMetrics: c.TaskHandlerPullRefMetrics, diff --git a/pkg/controller/projects.go b/pkg/controller/projects.go index b21193be..1ba89bcf 100644 --- a/pkg/controller/projects.go +++ b/pkg/controller/projects.go @@ -9,6 +9,37 @@ import ( "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" ) +// PullProject .. +func (c *Controller) PullProject(ctx context.Context, name string) error { + gp, err := c.Gitlab.GetProject(ctx, name) + if err != nil { + return err + } + p := schemas.NewProject(gp.PathWithNamespace) + + projectExists, err := c.Store.ProjectExists(ctx, p.Key()) + if err != nil { + return err + } + + if !projectExists { + log.WithFields(log.Fields{ + "project-name": p.Name, + }).Info("discovered new project") + + if err := c.Store.SetProject(ctx, p); err != nil { + log.WithContext(ctx). + WithError(err). + Error() + } + + c.ScheduleTask(ctx, schemas.TaskTypePullRefsFromProject, string(p.Key()), p) + c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentsFromProject, string(p.Key()), p) + } + + return nil +} + // PullProjectsFromWildcard .. func (c *Controller) PullProjectsFromWildcard(ctx context.Context, w config.Wildcard) error { foundProjects, err := c.Gitlab.ListProjects(ctx, w) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 37852af6..0430caef 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -71,6 +71,13 @@ func NewTaskController(ctx context.Context, r *redis.Client, maximumJobsQueueSiz return } +// TaskHandlerPullProject .. +func (c *Controller) TaskHandlerPullProject(ctx context.Context, name string) error { + defer c.unqueueTask(ctx, schemas.TaskTypePullProject, name) + + return c.PullProject(ctx, name) +} + // TaskHandlerPullProjectsFromWildcard .. func (c *Controller) TaskHandlerPullProjectsFromWildcard(ctx context.Context, id string, w config.Wildcard) error { defer c.unqueueTask(ctx, schemas.TaskTypePullProjectsFromWildcard, id) diff --git a/pkg/controller/webhooks.go b/pkg/controller/webhooks.go index 11b2efc0..4f6386bd 100644 --- a/pkg/controller/webhooks.go +++ b/pkg/controller/webhooks.go @@ -155,9 +155,9 @@ func (c *Controller) triggerRefMetricsPull(ctx context.Context, ref schemas.Ref) // Perhaps the project is discoverable through a wildcard if !projectExists && len(c.Config.Wildcards) > 0 { - for id, w := range c.Config.Wildcards { + for _, w := range c.Config.Wildcards { // If in all our wildcards we have one which can potentially match the project ref - // received, we trigger a scan + // received, we trigger a pull of the project matches, err := isRefMatchingWilcard(w, ref) if err != nil { log.WithContext(ctx). @@ -168,8 +168,8 @@ func (c *Controller) triggerRefMetricsPull(ctx context.Context, ref schemas.Ref) } if matches { - c.ScheduleTask(context.TODO(), schemas.TaskTypePullProjectsFromWildcard, strconv.Itoa(id), strconv.Itoa(id), w) - log.WithFields(logFields).Info("project ref not currently exported but its configuration matches a wildcard, triggering a pull of the projects from this wildcard") + c.ScheduleTask(context.TODO(), schemas.TaskTypePullProject, ref.Project.Name) + log.WithFields(logFields).Info("project ref not currently exported but its configuration matches a wildcard, triggering a pull of the project") } else { log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..") } @@ -269,9 +269,9 @@ func (c *Controller) triggerEnvironmentMetricsPull(ctx context.Context, env sche // Perhaps the project is discoverable through a wildcard if !projectExists && len(c.Config.Wildcards) > 0 { - for id, w := range c.Config.Wildcards { + for _, w := range c.Config.Wildcards { // If in all our wildcards we have one which can potentially match the env - // received, we trigger a scan + // received, we trigger a pull of the project matches, err := isEnvMatchingWilcard(w, env) if err != nil { log.WithContext(ctx). @@ -282,8 +282,8 @@ func (c *Controller) triggerEnvironmentMetricsPull(ctx context.Context, env sche } if matches { - c.ScheduleTask(ctx, schemas.TaskTypePullProjectsFromWildcard, strconv.Itoa(id), strconv.Itoa(id), w) - log.WithFields(logFields).Info("project environment not currently exported but its configuration matches a wildcard, triggering a pull of the projects from this wildcard") + c.ScheduleTask(context.TODO(), schemas.TaskTypePullProject, env.ProjectName) + log.WithFields(logFields).Info("project environment not currently exported but its configuration matches a wildcard, triggering a pull of the project") } else { log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..") } diff --git a/pkg/schemas/tasks.go b/pkg/schemas/tasks.go index 932b0786..1be9232c 100644 --- a/pkg/schemas/tasks.go +++ b/pkg/schemas/tasks.go @@ -4,6 +4,9 @@ package schemas type TaskType string const ( + // TaskTypePullProject .. + TaskTypePullProject TaskType = "PullProject" + // TaskTypePullProjectsFromWildcard .. TaskTypePullProjectsFromWildcard TaskType = "PullProjectsFromWildcard"