From 2709176dcc8d258fb406d9c1a84b94c07a76c1c8 Mon Sep 17 00:00:00 2001 From: Juliano Martinez Date: Fri, 20 Dec 2024 12:43:13 +0100 Subject: [PATCH] Implement AccessorID resolution and client ip propagation --- admissionctrl/controller.go | 20 +-- admissionctrl/mutator/json_patch_webhook.go | 12 ++ admissionctrl/mutator/webhook_mutator.go | 12 ++ admissionctrl/validator/webhook_validator.go | 12 ++ cmd/nacp/nacp.go | 123 ++++++++++++++++--- cmd/nacp/nacp_test.go | 79 +++++++++--- config/config.go | 23 ++-- 7 files changed, 233 insertions(+), 48 deletions(-) diff --git a/admissionctrl/controller.go b/admissionctrl/controller.go index ac94f0f..bcfafe4 100644 --- a/admissionctrl/controller.go +++ b/admissionctrl/controller.go @@ -27,16 +27,18 @@ type JobValidator interface { } type JobHandler struct { - mutators []JobMutator - validators []JobValidator - logger hclog.Logger + mutators []JobMutator + validators []JobValidator + resolveToken bool + logger hclog.Logger } -func NewJobHandler(mutators []JobMutator, validators []JobValidator, logger hclog.Logger) *JobHandler { +func NewJobHandler(mutators []JobMutator, validators []JobValidator, logger hclog.Logger, resolverToken bool) *JobHandler { return &JobHandler{ - mutators: mutators, - validators: validators, - logger: logger, + mutators: mutators, + validators: validators, + logger: logger, + resolveToken: resolverToken, } } @@ -97,6 +99,10 @@ func (j *JobHandler) AdmissionValidators(origJob *api.Job) ([]error, error) { } +func (j *JobHandler) ResolveToken() bool { + return j.resolveToken +} + func copyJob(job *api.Job) *api.Job { jobCopy := &api.Job{} data, err := json.Marshal(job) diff --git a/admissionctrl/mutator/json_patch_webhook.go b/admissionctrl/mutator/json_patch_webhook.go index dfe214a..39ac93c 100644 --- a/admissionctrl/mutator/json_patch_webhook.go +++ b/admissionctrl/mutator/json_patch_webhook.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/mxab/nacp/config" "net/http" "net/url" @@ -48,6 +49,17 @@ func (j *JsonPatchWebhookMutator) Mutate(job *api.Job) (*api.Job, []error, error if err != nil { return nil, nil, err } + // Add context headers if available + if ctx := req.Context(); ctx != nil { + if reqCtx, ok := ctx.Value("request_context").(*config.RequestContext); ok { + if reqCtx.ClientIP != "" { + req.Header.Set("X-Forwarded-For", reqCtx.ClientIP) + } + if reqCtx.AccessorID != "" { + req.Header.Set("X-Nomad-Accessor-ID", reqCtx.AccessorID) + } + } + } res, err := httpClient.Do(req) if err != nil { return nil, nil, err diff --git a/admissionctrl/mutator/webhook_mutator.go b/admissionctrl/mutator/webhook_mutator.go index fa2848f..6bbe4d9 100644 --- a/admissionctrl/mutator/webhook_mutator.go +++ b/admissionctrl/mutator/webhook_mutator.go @@ -3,6 +3,7 @@ package mutator import ( "bytes" "encoding/json" + "github.com/mxab/nacp/config" "net/http" "net/url" @@ -27,6 +28,17 @@ func (w *WebhookMutator) Mutate(job *api.Job) (out *api.Job, warnings []error, e if err != nil { return nil, nil, err } + // Add context headers if available + if ctx := req.Context(); ctx != nil { + if reqCtx, ok := ctx.Value("request_context").(*config.RequestContext); ok { + if reqCtx.ClientIP != "" { + req.Header.Set("X-Forwarded-For", reqCtx.ClientIP) + } + if reqCtx.AccessorID != "" { + req.Header.Set("X-Nomad-Accessor-ID", reqCtx.AccessorID) + } + } + } req.Header.Set("Content-Type", "application/json") client := &http.Client{} diff --git a/admissionctrl/validator/webhook_validator.go b/admissionctrl/validator/webhook_validator.go index 2397a1b..30d477b 100644 --- a/admissionctrl/validator/webhook_validator.go +++ b/admissionctrl/validator/webhook_validator.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/mxab/nacp/config" "net/http" "net/url" @@ -34,6 +35,17 @@ func (w *WebhookValidator) Validate(job *api.Job) ([]error, error) { if err != nil { return nil, err } + // Add context headers if available + if ctx := req.Context(); ctx != nil { + if reqCtx, ok := ctx.Value("request_context").(*config.RequestContext); ok { + if reqCtx.ClientIP != "" { + req.Header.Set("X-Forwarded-For", reqCtx.ClientIP) + } + if reqCtx.AccessorID != "" { + req.Header.Set("X-Nomad-Accessor-ID", reqCtx.AccessorID) + } + } + } resp, err := http.DefaultClient.Do(req) if err != nil { diff --git a/cmd/nacp/nacp.go b/cmd/nacp/nacp.go index 96b80fc..d27497e 100644 --- a/cmd/nacp/nacp.go +++ b/cmd/nacp/nacp.go @@ -17,6 +17,7 @@ import ( "os" "regexp" "strconv" + "strings" "time" "github.com/hashicorp/go-hclog" @@ -44,6 +45,62 @@ var ( nomadTimeout = 310 * time.Second ) +// New function to get client IP +func getClientIP(r *http.Request) string { + // Check X-Forwarded-For header first + forwarded := r.Header.Get("X-Forwarded-For") + if forwarded != "" { + return strings.Split(forwarded, ",")[0] + } + + // Fall back to RemoteAddr + ip, _, _ := net.SplitHostPort(r.RemoteAddr) + return ip +} + +func resolveTokenAccessor(transport *http.Transport, nomadAddress *url.URL, token string) (string, error) { + if token == "" { + return "", nil + } + + client := &http.Client{ + Transport: transport, + } + if transport == nil { + client = http.DefaultClient + } + + selfURL := *nomadAddress + selfURL.Path = "/v1/acl/token/self" + + req, err := http.NewRequest("GET", selfURL.String(), nil) + if err != nil { + return "", err + } + + req.Header.Set("X-Nomad-Token", token) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to resolve token: %s", resp.Status) + } + + var tokenInfo struct { + AccessorID string `json:"AccessorID"` + } + + if err := json.NewDecoder(resp.Body).Decode(&tokenInfo); err != nil { + return "", err + } + + return tokenInfo.AccessorID, nil +} + func NewProxyHandler(nomadAddress *url.URL, jobHandler *admissionctrl.JobHandler, appLogger hclog.Logger, transport *http.Transport) func(http.ResponseWriter, *http.Request) { proxy := httputil.NewSingleHostReverseProxy(nomadAddress) @@ -80,6 +137,25 @@ func NewProxyHandler(nomadAddress *url.URL, jobHandler *admissionctrl.JobHandler appLogger.Info("Request received", "path", r.URL.Path, "method", r.Method) + ctx := r.Context() + reqCtx := &config.RequestContext{ + ClientIP: getClientIP(r), + } + + token := r.Header.Get("X-Nomad-Token") + if jobHandler.ResolveToken() { + accessorID, err := resolveTokenAccessor(transport, nomadAddress, token) + if err != nil { + appLogger.Error("Resolving token failed", "error", err) + writeError(w, err) + } + reqCtx.AccessorID = accessorID + } + + // Store context + ctx = context.WithValue(ctx, "request_context", reqCtx) + r = r.WithContext(ctx) + var err error //var err error if isRegister(r) { @@ -457,20 +533,28 @@ func buildServer(c *config.Config, appLogger hclog.Logger) (*http.Server, error) } proxyTransport.TLSClientConfig = nomadTlsConfig } - jobMutators, err := createMutators(c, appLogger.Named("mutators")) + + jobMutators, resolveTokenMutators, err := createMutators(c, appLogger.Named("mutators")) if err != nil { return nil, fmt.Errorf("failed to create mutators: %w", err) } - jobValidators, err := createValidators(c, appLogger.Named("validators")) + + jobValidators, resolveTokenValidators, err := createValidators(c, appLogger.Named("validators")) if err != nil { return nil, fmt.Errorf("failed to create validators: %w", err) } + var resolveToken bool + if resolveTokenMutators || resolveTokenValidators { + resolveToken = true + } + handler := admissionctrl.NewJobHandler( jobMutators, jobValidators, appLogger.Named("handler"), + resolveToken, ) proxy := NewProxyHandler(backend, handler, appLogger, proxyTransport) @@ -535,72 +619,79 @@ func createTlsConfig(caFile string, noClientCert bool) (*tls.Config, error) { return tlsConfig, nil } -func createMutators(c *config.Config, logger hclog.Logger) ([]admissionctrl.JobMutator, error) { +func createMutators(c *config.Config, logger hclog.Logger) ([]admissionctrl.JobMutator, bool, error) { var jobMutators []admissionctrl.JobMutator + var resolveToken bool for _, m := range c.Mutators { + if m.ResolveToken { + resolveToken = true + } switch m.Type { - case "opa_json_patch": notationVerifier, err := buildVerifierIfEnabled(m.OpaRule.Notation, logger.Named("notation_verifier")) if err != nil { - return nil, err + return nil, resolveToken, err } mutator, err := mutator.NewOpaJsonPatchMutator(m.Name, m.OpaRule.Filename, m.OpaRule.Query, logger.Named("opa_mutator"), notationVerifier) if err != nil { - return nil, err + return nil, resolveToken, err } jobMutators = append(jobMutators, mutator) case "json_patch_webhook": mutator, err := mutator.NewJsonPatchWebhookMutator(m.Name, m.Webhook.Endpoint, m.Webhook.Method, logger.Named("json_patch_webhook_mutator")) if err != nil { - return nil, err + return nil, resolveToken, err } jobMutators = append(jobMutators, mutator) default: - return nil, fmt.Errorf("unknown mutator type %s", m.Type) + return nil, resolveToken, fmt.Errorf("unknown mutator type %s", m.Type) } } - return jobMutators, nil + return jobMutators, resolveToken, nil } -func createValidators(c *config.Config, logger hclog.Logger) ([]admissionctrl.JobValidator, error) { +func createValidators(c *config.Config, logger hclog.Logger) ([]admissionctrl.JobValidator, bool, error) { var jobValidators []admissionctrl.JobValidator + var resolveToken bool for _, v := range c.Validators { + if v.ResolveToken { + resolveToken = true + } switch v.Type { case "opa": notationVerifier, err := buildVerifierIfEnabled(v.Notation, logger.Named("notation_verifier")) if err != nil { - return nil, err + return nil, resolveToken, err } opaValidator, err := validator.NewOpaValidator(v.Name, v.OpaRule.Filename, v.OpaRule.Query, logger.Named("opa_validator"), notationVerifier) if err != nil { - return nil, err + return nil, resolveToken, err } jobValidators = append(jobValidators, opaValidator) case "webhook": validator, err := validator.NewWebhookValidator(v.Name, v.Webhook.Endpoint, v.Webhook.Method, logger.Named("webhook_validator")) if err != nil { - return nil, err + return nil, resolveToken, err } jobValidators = append(jobValidators, validator) case "notation": notationVerifier, err := buildVerifier(v.Notation, logger.Named("notation_verifier")) if err != nil { - return nil, err + return nil, resolveToken, err } validator := validator.NewNotationValidator(logger.Named("notation_validator"), v.Name, notationVerifier) jobValidators = append(jobValidators, validator) default: - return nil, fmt.Errorf("unknown validator type %s", v.Type) + return nil, resolveToken, fmt.Errorf("unknown validator type %s", v.Type) } } - return jobValidators, nil + return jobValidators, resolveToken, nil } func buildVerifierIfEnabled(notationVerifierConfig *config.NotationVerifierConfig, logger hclog.Logger) (notation.ImageVerifier, error) { if notationVerifierConfig == nil { diff --git a/cmd/nacp/nacp_test.go b/cmd/nacp/nacp_test.go index fb390dc..b5df01d 100644 --- a/cmd/nacp/nacp_test.go +++ b/cmd/nacp/nacp_test.go @@ -40,6 +40,10 @@ func TestProxy(t *testing.T) { path string method string + token string + resolveToken bool + accessorID string + requestSender func(*api.Client) (interface{}, *api.WriteMeta, error) wantNomadRequestJson string wantProxyResponse interface{} @@ -305,6 +309,31 @@ func TestProxy(t *testing.T) { Warnings: helper.MergeMultierrorWarnings(errors.New("some warning")), }, + nomadResponse: toJson(t, &api.JobValidateResponse{}), + nomadResponseEncoding: "gzip", + validators: []admissionctrl.JobValidator{ + mockValidatorReturningWarnings("some warning"), + }, + mutators: []admissionctrl.JobMutator{}, + }, + { + name: "resolves token during job creation", + path: "/v1/validate/job", + + method: "PUT", + token: "test-token", + resolveToken: true, + accessorID: "test-accessor", + + requestSender: func(c *api.Client) (interface{}, *api.WriteMeta, error) { + return c.Jobs().Validate(&api.Job{}, nil) + }, + wantNomadRequestJson: toJson(t, &api.JobValidateRequest{Job: &api.Job{}}), + + wantProxyResponse: &api.JobValidateResponse{ + Warnings: helper.MergeMultierrorWarnings(errors.New("some warning")), + }, + nomadResponse: toJson(t, &api.JobValidateResponse{}), nomadResponseEncoding: "gzip", validators: []admissionctrl.JobValidator{ @@ -317,38 +346,43 @@ func TestProxy(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { nomadBackendCalled := false + tokenCalled := false nomadDummy := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - // Test request parameters nomadBackendCalled = true - assert.Equal(t, req.Method, tc.method, "Ensure method is set") - assert.Equal(t, req.URL.Path, tc.path, "Ensure path is set") + if req.URL.Path == "/v1/acl/token/self" { + tokenCalled = true + if tc.token != "test-token" { + rw.WriteHeader(http.StatusUnauthorized) + return + } + json.NewEncoder(rw).Encode(map[string]string{ + "AccessorID": tc.accessorID, + }) + return + } + + assert.Equal(t, req.Method, tc.method) + assert.Equal(t, req.URL.Path, tc.path) jsonData, err := io.ReadAll(req.Body) if err != nil { t.Fatal(err) } json := string(jsonData) - assert.JSONEq(t, tc.wantNomadRequestJson, json, "Body matches") + assert.JSONEq(t, tc.wantNomadRequestJson, json) - //set encoding to gzip if tc.nomadResponseEncoding == "gzip" { rw.Header().Set("Content-Encoding", "gzip") rw.WriteHeader(http.StatusOK) - //write gzip response gzipWriter := gzip.NewWriter(rw) defer gzipWriter.Close() gzipWriter.Write([]byte(tc.nomadResponse)) - } else { rw.WriteHeader(http.StatusOK) rw.Write([]byte(tc.nomadResponse)) } - })) - // Close the server when test finishes defer nomadDummy.Close() - // Use Client & URL from our local test server - nomad, err := url.Parse(nomadDummy.URL) if err != nil { t.Fatal(err) @@ -357,24 +391,33 @@ func TestProxy(t *testing.T) { tc.mutators, tc.validators, hclog.NewNullLogger(), + tc.resolveToken, ) - proxy := NewProxyHandler(nomad, jobHandler, hclog.NewNullLogger(), nil) + proxy := NewProxyHandler(nomad, jobHandler, hclog.NewNullLogger(), nil) proxyServer := httptest.NewServer(http.HandlerFunc(proxy)) defer proxyServer.Close() nomadClient := buildNomadClient(t, proxyServer) + if tc.token != "" { + nomadClient.SetSecretID(tc.token) + } + resp, _, err := tc.requestSender(nomadClient) assert.NoError(t, err, "No http call error") assert.Equal(t, tc.wantProxyResponse, resp, "OK response is expected") - assert.True(t, nomadBackendCalled, "Nomad backend was called") + if tc.resolveToken { + assert.True(t, tokenCalled, "Token resolution should be called") + } else { + assert.False(t, tokenCalled, "Token resolution should not be called") + } }) } - } + func TestJobUpdateProxy(t *testing.T) { type test struct { @@ -440,6 +483,7 @@ func TestJobUpdateProxy(t *testing.T) { tc.mutators, tc.validators, hclog.NewNullLogger(), + false, ) proxy := NewProxyHandler(nomad, jobHandler, hclog.NewNullLogger(), nil) @@ -540,6 +584,7 @@ func TestAdmissionControllerErrors(t *testing.T) { []admissionctrl.JobMutator{}, []admissionctrl.JobValidator{validator}, hclog.NewNullLogger(), + false, ) proxy := NewProxyHandler(nomad, jobHandler, hclog.NewNullLogger(), nil) @@ -657,7 +702,7 @@ func TestCreateValidators(t *testing.T) { Validators: []config.Validator{tc.validators}, } - validators, err := createValidators(c, hclog.NewNullLogger()) + validators, _, err := createValidators(c, hclog.NewNullLogger()) if tc.wantErr { assert.Error(t, err) @@ -715,7 +760,7 @@ func TestNotationValidatorConfig(t *testing.T) { }, } - validators, err := createValidators(c, hclog.NewNullLogger()) + validators, _, err := createValidators(c, hclog.NewNullLogger()) assert.NoError(t, err) assert.IsType(t, &validator.NotationValidator{}, validators[0]) @@ -776,7 +821,7 @@ func TestCreateMutatators(t *testing.T) { Mutators: []config.Mutator{tc.mutators}, } - mutators, err := createMutators(c, hclog.NewNullLogger()) + mutators, _, err := createMutators(c, hclog.NewNullLogger()) if tc.wantErr { assert.Error(t, err) diff --git a/config/config.go b/config/config.go index 7969a92..8757eb3 100644 --- a/config/config.go +++ b/config/config.go @@ -16,18 +16,25 @@ type OpaRule struct { } type Validator struct { - Type string `hcl:"type,label"` - Name string `hcl:"name,label"` - OpaRule *OpaRule `hcl:"opa_rule,block"` - Webhook *Webhook `hcl:"webhook,block"` + Type string `hcl:"type,label"` + Name string `hcl:"name,label"` + OpaRule *OpaRule `hcl:"opa_rule,block"` + Webhook *Webhook `hcl:"webhook,block"` + ResolveToken bool `hcl:"resolve_token,optional"` Notation *NotationVerifierConfig `hcl:"notation,block"` } type Mutator struct { - Type string `hcl:"type,label"` - Name string `hcl:"name,label"` - OpaRule *OpaRule `hcl:"opa_rule,block"` - Webhook *Webhook `hcl:"webhook,block"` + Type string `hcl:"type,label"` + Name string `hcl:"name,label"` + OpaRule *OpaRule `hcl:"opa_rule,block"` + Webhook *Webhook `hcl:"webhook,block"` + ResolveToken bool `hcl:"resolve_token,optional"` +} +type RequestContext struct { + ClientIP string + AccessorID string + ResolveToken bool } type NomadServerTLS struct {