Feat: add support for "include" config from other repos (#74)

* feat: initial work on include config support

* docs: document the new 'include' settings

* docs: tweak docs

cmd/shared.go

@@ -113,6 +113,11 @@ func ProcessMR(ctx context.Context, client scm.Client, cfg *config.Config, event
 		return errors.New("cfg==nil; this is unexpected an error, please report!")
 	}
 
+	// Load any remote configuration files
+	if err := cfg.LoadIncludes(ctx, client); err != nil {
+		return fmt.Errorf("failed to load 'include' settings: %w", err)
+	}
+
 	// Allow changing the 'dry-run' mode via configuration file
 	if cfg.DryRun != nil && *cfg.DryRun != state.IsDryRun(ctx) {
 		slogctx.Info(ctx, "Configuration file has a 'dry_run' value, using that in favor of server default")

docs/configuration.md

@@ -8,9 +8,9 @@ The file path can be changed via `--config` CLI flag and `#!css $SCM_ENGINE_CONF
 
 !!! question "What is 'activity'?"
 
-  SCM-Engine defines activity as comments, reviews, commits, adding/removing labels and similar actions made on a change request.
+    SCM-Engine defines activity as comments, reviews, commits, adding/removing labels and similar actions made on a change request.
 
-  *Generally*, `activity` is what you see in the Merge/Pull Request `timeline` in the browser UI.
+    *Generally*, `activity` is what you see in the Merge/Pull Request `timeline` in the browser UI.
 
 Configure what users that should be ignored when considering activity on a Merge Request
 
@@ -28,6 +28,57 @@ A list of emails that should be ignored when considering user activity. Default:
 
 **NOTE:** If a user do not have a public email configured on their profile, that users activity will never match this rule.
 
+## `include[]` {#include data-toc-label="include"}
+
+!!! question "What are includes?"
+
+    `scm-engine` has support for importing some (or all) of its configuration from other repositories.
+
+!!! note "The `scm-engine` API token MUST be able to read the content of the referenced projects via the API"
+
+!!! abstract "Limitations and restrictions of remote included configuration files"
+
+    This is immensely useful if you want to share configuration between many projects, like a centralized `scm-engine-library` project with common patterns and configuration files.
+
+    * Only `actions` and `label` configurations keys are supported in included configuration files.
+    * Nested/Recursive includes are NOT support.
+    * Merging/overriding configurations are NOT supported; included configuration will always append to the existing configuration.
+    * All included files MUST exist and be valid; any missing file or invalid configuration will result in failure.
+    * `scm-engine` will read all files from a project in a single request where possible; up to 100 files are supported.
+    * `scm-engine` do NOT cache any remote configuration files; they are always read during evaluation cycle.
+
+!!! example "Example 'include' configuration loading 4 files from the 'platform/scm-engine-library' project"
+
+    ```yaml
+    include:
+      - project: platform/scm-engine-library
+        files:
+          - label/change-type.yml
+          - label/last-commit-age.yml
+          - label/need-rebase.yml
+          - life-cycle/close-merge-request-3-weeks.yml
+
+      label:
+        - ....
+
+      actions:
+        - ...
+    ```
+
+### `include[].project` {#include.project data-toc-label="project"}
+
+The GitLab repository slug to read configuration files, like `example/project`.
+
+### `include[].files` {#include.files data-toc-label="files"}
+
+The list of files to include from the project. The paths must be *relative* to the repository root, e.x. `label/some-config-file.yml`; NOT `/label/some-config-file.yml`
+
+### `include[].ref` {#include.ref data-toc-label="ref"}
+
+Optional Git reference to read the configuration from; it can be a tag, branch, or commit SHA.
+
+If omitted, `HEAD` is used; meaning your default branch.
+
 ## `actions[]` {#actions data-toc-label="actions"}
 
 !!! question "What are actions?"
@@ -62,11 +113,11 @@ The list of operations to take if the [`#!css action.if`](#actions.if) returned
 
 This key controls what kind of action that should be taken.
 
-- `#!yaml approve` to approve the Merge Request.
-- `#!yaml unapprove` to approve the Merge Request.
-- `#!yaml close` to close the Merge Request.
-- `#!yaml reopen` to reopen the Merge Request.
-- `#!yaml comment` to add a comment to the Merge Request
+* `#!yaml approve` to approve the Merge Request.
+* `#!yaml unapprove` to approve the Merge Request.
+* `#!yaml close` to close the Merge Request.
+* `#!yaml reopen` to reopen the Merge Request.
+* `#!yaml comment` to add a comment to the Merge Request
 
       *Additional fields:*
 
@@ -78,9 +129,9 @@ This key controls what kind of action that should be taken.
           Hello world
       ```
 
-- `#!yaml lock_discussion` to prevent further discussions on the Merge Request.
-- `#!yaml unlock_discussion` to allow discussions on the Merge Request.
-- `#!yaml add_label` to add *an existing* label to the Merge Request
+* `#!yaml lock_discussion` to prevent further discussions on the Merge Request.
+* `#!yaml unlock_discussion` to allow discussions on the Merge Request.
+* `#!yaml add_label` to add *an existing* label to the Merge Request
 
       *Additional fields:*
 
@@ -91,7 +142,7 @@ This key controls what kind of action that should be taken.
         label: example
       ```
 
-- `#!yaml remove_label` to remove a label from the Merge Request
+* `#!yaml remove_label` to remove a label from the Merge Request
 
       *Additional fields:*
 
@@ -102,7 +153,7 @@ This key controls what kind of action that should be taken.
         label: example
       ```
 
-- `#!yaml update_description` updates the Merge Request Description
+* `#!yaml update_description` updates the Merge Request Description
 
       *Additional fields:*
 
@@ -129,11 +180,11 @@ These keys are shared between the [`#!yaml conditional`](#label.strategy-conditi
 
 SCM Engine supports two strategies for managing labels, each changes the behavior of the [`#!css script`](#label.script).
 
-- `#!yaml conditional` (default, if `#!css strategy` key is omitted), where you provide the `#!css name` of the label, and a [`#!css script`](#label.script) that returns a boolean for wether the label should be added to the Merge Request.
+* `#!yaml conditional` (default, if `#!css strategy` key is omitted), where you provide the `#!css name` of the label, and a [`#!css script`](#label.script) that returns a boolean for wether the label should be added to the Merge Request.
 
     The [`#!css script`](#label.script) must return a `#!yaml boolean` value, where `#!yaml true` mean `add the label` and `#!yaml false` mean `remove the label`.
 
-- `#!yaml generate`, where your `#!css script` generates the list of labels that should be added to the Merge Request.
+* `#!yaml generate`, where your `#!css script` generates the list of labels that should be added to the Merge Request.
 
     The [`#!css script`](#label.script) must return a `list of strings`, where each label returned will be added to the Merge Request.
 
@@ -153,11 +204,11 @@ Thanks to the dynamic nature of the `#!yaml generate` strategy, it has fantastic
 
 ### `label[].name` {#label.name data-toc-label="name"}
 
-- When using `#!yaml label.strategy: conditional`
+* When using `#!yaml label.strategy: conditional`
 
     **REQUIRED** The `#!css name` of the label to create.
 
-- When using `#!yaml label.strategy: generate`
+* When using `#!yaml label.strategy: generate`
 
     **OMITTED** The `#!css name` field must not be set when using the `#!yaml generate` strategy.
 

pkg/config/config.go

@@ -3,16 +3,18 @@ package config
 import (
 	"context"
 	"fmt"
+	"log/slog"
 
 	"github.com/jippi/scm-engine/pkg/scm"
 	slogctx "github.com/veqryn/slog-context"
 )
 
 type Config struct {
 	DryRun             *bool              `yaml:"dry_run"`
-	Labels             Labels             `yaml:"label"`
 	Actions            Actions            `yaml:"actions"`
 	IgnoreActivityFrom IgnoreActivityFrom `yaml:"ignore_activity_from"`
+	Includes           []Include          `yaml:"include"`
+	Labels             Labels             `yaml:"label"`
 }
 
 func (c Config) Evaluate(ctx context.Context, evalContext scm.EvalContext) ([]scm.EvaluationResult, []Action, error) {
@@ -32,3 +34,60 @@ func (c Config) Evaluate(ctx context.Context, evalContext scm.EvalContext) ([]sc
 
 	return labels, actions, nil
 }
+
+func (c *Config) LoadIncludes(ctx context.Context, client scm.Client) error {
+	// No files to include
+	if len(c.Includes) == 0 {
+		return nil
+	}
+
+	// Update logger with a friendly tag to differentiate the events within
+	ctx = slogctx.With(ctx, slog.String("phase", "remote_include"))
+
+	// For each project, do a read of all the files we need
+	for _, include := range c.Includes {
+		ctx := slogctx.With(ctx, slog.Any("remote_include_config", include))
+
+		slogctx.Debug(ctx, fmt.Sprintf("Loading remote configuration from project %q", include.Project))
+
+		files, err := client.GetProjectFiles(ctx, include.Project, include.Ref, include.Files)
+		if err != nil {
+			return fmt.Errorf("failed to load included config files from project [%s]: %w", include.Project, err)
+		}
+
+		for fileName, fileContent := range files {
+			remoteConfig, err := ParseFileString(fileContent)
+			if err != nil {
+				return fmt.Errorf("failed to parse remote config file [%s] from project [%s]: %w", fileName, include.Project, err)
+			}
+
+			// Disallow nested includes
+			if len(remoteConfig.Includes) != 0 {
+				slogctx.Warn(ctx, fmt.Sprintf("file [%s] from project [%s] may not have any 'include' settings; Recursive include is not supported", fileName, include.Project))
+			}
+
+			// Disallow changing dry run
+			if remoteConfig.DryRun != nil {
+				slogctx.Warn(ctx, fmt.Sprintf("file [%s] from project [%s] may not have a 'dry_run' setting; Remote include are not allowed to change this setting", fileName, include.Project))
+			}
+
+			// Append actions
+			if len(remoteConfig.Actions) != 0 {
+				slogctx.Debug(ctx, fmt.Sprintf("file [%s] from project [%s] added %d new actions to the config file", fileName, include.Project, len(remoteConfig.Actions)))
+
+				c.Actions = append(c.Actions, remoteConfig.Actions...)
+			}
+
+			// Append labels
+			if len(remoteConfig.Labels) != 0 {
+				slogctx.Debug(ctx, fmt.Sprintf("file [%s] from project [%s] added %d new labels to the config file", fileName, include.Project, len(remoteConfig.Labels)))
+
+				c.Labels = append(c.Labels, remoteConfig.Labels...)
+			}
+		}
+	}
+
+	slogctx.Debug(ctx, "Done loading remote configuration files")
+
+	return nil
+}

pkg/config/include.go

@@ -0,0 +1,7 @@
+package config
+
+type Include struct {
+	Project string   `yaml:"project"`
+	Ref     *string  `yaml:"ref"`
+	Files   []string `yaml:"files"`
+}

pkg/config/loader.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"io"
 	"os"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 )
@@ -33,3 +34,19 @@ func ParseFile(f io.Reader) (*Config, error) {
 
 	return config, nil
 }
+
+// ParseFile parses a Gitlabber file, returning a Config.
+func ParseFileString(in string) (*Config, error) {
+	config := &Config{}
+
+	buf := new(bytes.Buffer)
+	if _, err := buf.ReadFrom(strings.NewReader((in))); err != nil {
+		return nil, err
+	}
+
+	if err := yaml.Unmarshal(buf.Bytes(), config); err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}

pkg/scm/github/client.go

@@ -68,3 +68,8 @@ func (client *Client) Start(ctx context.Context) error {
 func (client *Client) Stop(ctx context.Context, err error) error {
 	return nil
 }
+
+// Get Project Files
+func (client *Client) GetProjectFiles(ctx context.Context, project string, ref *string, files []string) (map[string]string, error) {
+	return nil, errors.New("not implemented yet")
+}

pkg/scm/gitlab/client.go

@@ -2,6 +2,7 @@ package gitlab
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
@@ -62,23 +63,9 @@ func (client *Client) MergeRequests() scm.MergeRequestClient {
 // FindMergeRequestsForPeriodicEvaluation will find all Merge Requests legible for
 // periodic re-evaluation.
 func (client *Client) FindMergeRequestsForPeriodicEvaluation(ctx context.Context, filters scm.MergeRequestListFilters) ([]scm.PeriodicEvaluationMergeRequest, error) {
-	httpClient := oauth2.NewClient(
-		ctx,
-		oauth2.StaticTokenSource(
-			&oauth2.Token{
-				AccessToken: state.Token(ctx),
-			},
-		),
-	)
-
-	gClient := graphql.NewClient(
-		graphqlBaseURL(client.wrapped.BaseURL())+"/api/graphql",
-		httpClient,
-	)
-
 	var response PeriodicEvaluationResult
 
-	if err := gClient.Query(ctx, &response, filters.AsGraphqlVariables()); err != nil {
+	if err := client.newGraphQLClient(ctx).Query(ctx, &response, filters.AsGraphqlVariables()); err != nil {
 		return nil, err
 	}
 
@@ -113,6 +100,50 @@ func (client *Client) EvalContext(ctx context.Context) (scm.EvalContext, error)
 	return NewContext(ctx, graphqlBaseURL(client.wrapped.BaseURL()), state.Token(ctx))
 }
 
+func (client *Client) GetProjectFiles(ctx context.Context, project string, ref *string, files []string) (map[string]string, error) {
+	if len(project) == 0 {
+		return nil, errors.New("Missing required 'project' value for include")
+	}
+
+	if len(files) == 0 {
+		return nil, fmt.Errorf("Missing list of files to include from project [%s]", project)
+	}
+
+	var (
+		response  IncludeConfigurationResult
+		variables = map[string]any{
+			"project": graphql.ID(project),
+			"files":   files,
+			"ref":     ref,
+		}
+	)
+
+	if err := client.newGraphQLClient(ctx).Query(ctx, &response, variables); err != nil {
+		return nil, fmt.Errorf("GraphQL query failed while trying to read remote configuration files [%v] for project [%s]: %w", files, project, err)
+	}
+
+	fileContents := map[string]string{}
+
+	// Convert the GraphQL response into a simple map
+	for _, blob := range response.Project.Repository.Blobs.Nodes {
+		fileContents[blob.Path] = blob.Blob
+	}
+
+	// Check if the files provided as input all exist in the file content and is not empty
+	for _, file := range files {
+		val, ok := fileContents[file]
+		if !ok {
+			return nil, fmt.Errorf("configuration file [%s] in project [%s] does not exist (or could not be read)", file, project)
+		}
+
+		if len(val) == 0 {
+			return nil, fmt.Errorf("configuration file [%s] in project [%s] is empty", file, project)
+		}
+	}
+
+	return fileContents, nil
+}
+
 // Start pipeline
 func (client *Client) Start(ctx context.Context) error {
 	ok, pattern := state.ShouldUpdatePipeline(ctx)
@@ -237,3 +268,19 @@ func graphqlBaseURL(inputURL *url.URL) string {
 
 	return buf.String()
 }
+
+func (client Client) newGraphQLClient(ctx context.Context) *graphql.Client {
+	httpClient := oauth2.NewClient(
+		ctx,
+		oauth2.StaticTokenSource(
+			&oauth2.Token{
+				AccessToken: state.Token(ctx),
+			},
+		),
+	)
+
+	return graphql.NewClient(
+		graphqlBaseURL(client.wrapped.BaseURL())+"/api/graphql",
+		httpClient,
+	)
+}