From ffcc8bdc6a92cbd2be23926300a2219762485374 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 22 Jun 2023 14:18:47 +0200 Subject: [PATCH 01/56] fix(otelx): forward all headers (#698) Signed-off-by: David van der Spek --- otelx/jaeger.go | 2 ++ otelx/otlp.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/otelx/jaeger.go b/otelx/jaeger.go index 0ad07a2c..f58e3fc8 100644 --- a/otelx/jaeger.go +++ b/otelx/jaeger.go @@ -67,8 +67,10 @@ func SetupJaeger(t *Tracer, tracerName string, c *Config) (trace.Tracer, error) // Simply add propagation.TraceContext{} and propagation.Baggage{} // here to enable those as well. prop := propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, jaegerPropagator.Jaeger{}, b3.New(b3.WithInjectEncoding(b3.B3MultipleHeader|b3.B3SingleHeader)), + propagation.Baggage{}, ) otel.SetTextMapPropagator(prop) return tp.Tracer(tracerName), nil diff --git a/otelx/otlp.go b/otelx/otlp.go index 0484267f..8fdc40d1 100644 --- a/otelx/otlp.go +++ b/otelx/otlp.go @@ -6,6 +6,8 @@ package otelx import ( "context" + "go.opentelemetry.io/contrib/propagators/b3" + jaegerPropagator "go.opentelemetry.io/contrib/propagators/jaeger" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" @@ -50,6 +52,8 @@ func SetupOTLP(t *Tracer, tracerName string, c *Config) (trace.Tracer, error) { otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( propagation.TraceContext{}, + jaegerPropagator.Jaeger{}, + b3.New(b3.WithInjectEncoding(b3.B3MultipleHeader|b3.B3SingleHeader)), propagation.Baggage{}, )) From 0d30de80a1f29c3a00a78da7703b05cbbfdd60c2 Mon Sep 17 00:00:00 2001 From: Patrik Date: Mon, 26 Jun 2023 09:54:18 +0200 Subject: [PATCH 02/56] fix(pagination): add table name to fields in keyset pagination (#699) --- pagination/keysetpagination/paginator.go | 20 +++++++++++-------- pagination/keysetpagination/paginator_test.go | 14 ++++++------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/pagination/keysetpagination/paginator.go b/pagination/keysetpagination/paginator.go index 4a7241d4..c6ec4bd1 100644 --- a/pagination/keysetpagination/paginator.go +++ b/pagination/keysetpagination/paginator.go @@ -84,33 +84,33 @@ func (p *Paginator) ToOptions() []Option { } } -func (p *Paginator) multipleOrderFieldsQuery(q *pop.Query, idField string, cols map[string]*columns.Column, quote func(string) string) { +func (p *Paginator) multipleOrderFieldsQuery(q *pop.Query, idField string, cols map[string]*columns.Column, quoteAndContextualize func(string) string) { tokenParts := p.Token().Parse(idField) idValue := tokenParts[idField] column, ok := cols[p.additionalColumn.name] if !ok { - q.Where(fmt.Sprintf(`%s > ?`, quote(idField)), idValue) + q.Where(fmt.Sprintf(`%s > ?`, quoteAndContextualize(idField)), idValue) return } - quoteName := quote(column.Name) + quoteName := quoteAndContextualize(column.Name) value, ok := tokenParts[column.Name] if !ok { - q.Where(fmt.Sprintf(`%s > ?`, quote(idField)), idValue) + q.Where(fmt.Sprintf(`%s > ?`, quoteAndContextualize(idField)), idValue) return } sign, keyword, err := p.additionalColumn.order.extract() if err != nil { - q.Where(fmt.Sprintf(`%s > ?`, quote(idField)), idValue) + q.Where(fmt.Sprintf(`%s > ?`, quoteAndContextualize(idField)), idValue) return } q. - Where(fmt.Sprintf("(%s %s ? OR (%s = ? AND %s > ?))", quoteName, sign, quoteName, quote(idField)), value, value, idValue). + Where(fmt.Sprintf("(%s %s ? OR (%s = ? AND %s > ?))", quoteName, sign, quoteName, quoteAndContextualize(idField)), value, value, idValue). Order(fmt.Sprintf("%s %s", quoteName, keyword)) } @@ -130,10 +130,14 @@ func Paginate[I any, PI interface { }](p *Paginator) pop.ScopeFunc { model := pop.Model{Value: new(I)} id := model.IDField() + tableName := model.Alias() return func(q *pop.Query) *pop.Query { - eid := q.Connection.Dialect.Quote(id) + eid := q.Connection.Dialect.Quote(tableName + "." + id) - p.multipleOrderFieldsQuery(q, id, model.Columns().Cols, q.Connection.Dialect.Quote) + quoteAndContextualize := func(name string) string { + return q.Connection.Dialect.Quote(tableName + "." + name) + } + p.multipleOrderFieldsQuery(q, id, model.Columns().Cols, quoteAndContextualize) return q. Limit(p.Size() + 1). diff --git a/pagination/keysetpagination/paginator_test.go b/pagination/keysetpagination/paginator_test.go index 1c17c069..d4c5c457 100644 --- a/pagination/keysetpagination/paginator_test.go +++ b/pagination/keysetpagination/paginator_test.go @@ -35,7 +35,7 @@ func TestPaginator(t *testing.T) { q = q.Scope(Paginate[testItem](paginator)) sql, args := q.ToSQL(&pop.Model{Value: new(testItem)}) - assert.Equal(t, "SELECT test_items.created_at, test_items.pk FROM test_items AS test_items WHERE \"pk\" > $1 ORDER BY \"pk\" ASC LIMIT 11", sql) + assert.Equal(t, `SELECT test_items.created_at, test_items.pk FROM test_items AS test_items WHERE "test_items"."pk" > $1 ORDER BY "test_items"."pk" ASC LIMIT 11`, sql) assert.Equal(t, []interface{}{"token"}, args) }) @@ -49,7 +49,7 @@ func TestPaginator(t *testing.T) { q = q.Scope(Paginate[testItem](paginator)) sql, args := q.ToSQL(&pop.Model{Value: new(testItem)}) - assert.Equal(t, "SELECT test_items.created_at, test_items.pk FROM test_items AS test_items WHERE `pk` > ? ORDER BY `pk` ASC LIMIT 11", sql) + assert.Equal(t, "SELECT test_items.created_at, test_items.pk FROM test_items AS test_items WHERE `test_items.pk` > ? ORDER BY `test_items.pk` ASC LIMIT 11", sql) assert.Equal(t, []interface{}{"token"}, args) }) @@ -185,31 +185,31 @@ func TestPaginateWithAdditionalColumn(t *testing.T) { { d: "with sort by created_at DESC", opts: []Option{WithToken(MapPageToken{"pk": "token_value", "created_at": "timestamp"}), WithColumn("created_at", "DESC")}, - e: `WHERE ("created_at" < $1 OR ("created_at" = $2 AND "pk" > $3)) ORDER BY "created_at" DESC, "pk" ASC`, + e: `WHERE ("test_items"."created_at" < $1 OR ("test_items"."created_at" = $2 AND "test_items"."pk" > $3)) ORDER BY "test_items"."created_at" DESC, "test_items"."pk" ASC`, args: []interface{}{"timestamp", "timestamp", "token_value"}, }, { d: "with sort by created_at ASC", opts: []Option{WithToken(MapPageToken{"pk": "token_value", "created_at": "timestamp"}), WithColumn("created_at", "ASC")}, - e: `WHERE ("created_at" > $1 OR ("created_at" = $2 AND "pk" > $3)) ORDER BY "created_at" ASC, "pk" ASC`, + e: `WHERE ("test_items"."created_at" > $1 OR ("test_items"."created_at" = $2 AND "test_items"."pk" > $3)) ORDER BY "test_items"."created_at" ASC, "test_items"."pk" ASC`, args: []interface{}{"timestamp", "timestamp", "token_value"}, }, { d: "with unknown column", opts: []Option{WithToken(MapPageToken{"pk": "token_value", "created_at": "timestamp"}), WithColumn("unknown_column", "ASC")}, - e: `WHERE "pk" > $1 ORDER BY "pk"`, + e: `WHERE "test_items"."pk" > $1 ORDER BY "test_items"."pk"`, args: []interface{}{"token_value"}, }, { d: "with no token value", opts: []Option{WithToken(MapPageToken{"pk": "token_value"}), WithColumn("created_at", "ASC")}, - e: `WHERE "pk" > $1 ORDER BY "pk"`, + e: `WHERE "test_items"."pk" > $1 ORDER BY "test_items"."pk"`, args: []interface{}{"token_value"}, }, { d: "with unknown order", opts: []Option{WithToken(MapPageToken{"pk": "token_value", "created_at": "timestamp"}), WithColumn("created_at", Order("unknown order"))}, - e: `WHERE "pk" > $1 ORDER BY "pk"`, + e: `WHERE "test_items"."pk" > $1 ORDER BY "test_items"."pk"`, args: []interface{}{"token_value"}, }, } { From 1c62a8a22f61052675b5cefe1ad45cb654677eaa Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Mon, 26 Jun 2023 11:37:58 +0200 Subject: [PATCH 03/56] feat: allow Go migrations outside of an SQL transaction (#696) Co-authored-by: aeneasr <3372410+aeneasr@users.noreply.github.com> --- popx/migration_box.go | 8 ++ popx/migration_box_gomigration_test.go | 132 +++++++++++++++++++++++++ popx/migration_info.go | 26 +++-- popx/migrator.go | 84 ++++++++++------ 4 files changed, 212 insertions(+), 38 deletions(-) diff --git a/popx/migration_box.go b/popx/migration_box.go index 740e3a49..f5919f6d 100644 --- a/popx/migration_box.go +++ b/popx/migration_box.go @@ -249,5 +249,13 @@ func (fm *MigrationBox) check() error { return errors.Errorf("migration %s has no corresponding down migration", up.Version) } } + + for _, m := range fm.Migrations { + for _, n := range m { + if err := n.Valid(); err != nil { + return err + } + } + } return nil } diff --git a/popx/migration_box_gomigration_test.go b/popx/migration_box_gomigration_test.go index cbe5470d..b685c6f0 100644 --- a/popx/migration_box_gomigration_test.go +++ b/popx/migration_box_gomigration_test.go @@ -6,6 +6,7 @@ package popx_test import ( "context" "database/sql" + "math/rand" "testing" "time" @@ -160,3 +161,134 @@ func TestGoMigrations(t *testing.T) { assert.ErrorIs(t, c.Where("i=1").First(tt), sql.ErrNoRows, "%+v", tt) }) } + +func TestIncompatibleRunners(t *testing.T) { + mb, err := popx.NewMigrationBox(empty, popx.NewMigrator(nil, logrusx.New("", ""), nil, 0), popx.WithGoMigrations( + popx.Migrations{ + { + Path: "transactional", + Version: "1", + Name: "gomigration_tx", + Direction: "up", + Type: "go", + DBType: "all", + RunnerNoTx: func(m popx.Migration, c *pop.Connection) error { + return nil + }, + Runner: func(m popx.Migration, c *pop.Connection, tx *pop.Tx) error { + return nil + }, + }, + { + Path: "transactional", + Version: "1", + Name: "gomigration_tx", + Direction: "down", + Type: "go", + DBType: "all", + RunnerNoTx: func(m popx.Migration, c *pop.Connection) error { + return nil + }, + }, + })) + require.ErrorContains(t, err, "incompatible transaction and non-transaction runners defined") + require.Nil(t, mb) + + mb, err = popx.NewMigrationBox(empty, popx.NewMigrator(nil, logrusx.New("", ""), nil, 0), popx.WithGoMigrations( + popx.Migrations{ + { + Path: "transactional", + Version: "1", + Name: "gomigration_tx", + Direction: "up", + Type: "go", + DBType: "all", + RunnerNoTx: nil, + Runner: nil, + }, + { + Path: "transactional", + Version: "1", + Name: "gomigration_tx", + Direction: "down", + Type: "go", + DBType: "all", + RunnerNoTx: nil, + Runner: nil, + }, + })) + require.ErrorContains(t, err, "no runner defined") + require.Nil(t, mb) +} + +func TestNoTransaction(t *testing.T) { + c, err := pop.NewConnection(&pop.ConnectionDetails{ + URL: "sqlite://file::memory:", + }) + require.NoError(t, err) + require.NoError(t, c.Open()) + + require.NoError(t, c.RawQuery("CREATE TABLE tests (i INTEGER, j INTEGER)").Exec()) + + up1, up2 := make(chan struct{}), make(chan struct{}) + down1, down2 := make(chan struct{}), make(chan struct{}) + rnd := rand.NewSource(time.Now().Unix()) + i1, i2, j1, j2 := rnd.Int63(), rnd.Int63(), rnd.Int63(), rnd.Int63() + mb, err := popx.NewMigrationBox(empty, popx.NewMigrator(c, logrusx.New("", ""), nil, 0), popx.WithGoMigrations( + popx.Migrations{ + { + Path: "gomigration_notx", + Version: "1", + Name: "gomigration no transaction", + Direction: "up", + Type: "go", + DBType: "all", + RunnerNoTx: func(m popx.Migration, c *pop.Connection) error { + if _, err := c.Store.Exec("INSERT INTO tests (i, j) VALUES (?, ?)", i1, j1); err != nil { + return errors.WithStack(err) + } + close(up1) + <-up2 + return nil + }, + }, + { + Path: "gomigration_notx", + Version: "1", + Name: "gomigration no transaction", + Direction: "down", + Type: "go", + DBType: "all", + RunnerNoTx: func(m popx.Migration, c *pop.Connection) error { + if _, err := c.Store.Exec("INSERT INTO tests (i, j) VALUES (?, ?)", i2, j2); err != nil { + return errors.WithStack(err) + } + close(down1) + <-down2 + return nil + }, + }, + }, + )) + require.NoError(t, err) + errs := make(chan error, 10) + go func() { + errs <- mb.Up(context.Background()) + }() + <-up1 + var j int64 + require.NoError(t, c.Store.Get(&j, "SELECT j FROM tests WHERE i = ?", i1)) + assert.Equal(t, j1, j) + close(up2) + assert.NoError(t, <-errs) + + go func() { + errs <- mb.Down(context.Background(), 20) + }() + <-down1 + j = 0 + require.NoError(t, c.Store.Get(&j, "SELECT j FROM tests WHERE i = ?", i2)) + assert.Equal(t, j2, j) + close(down2) + assert.NoError(t, <-errs) +} diff --git a/popx/migration_info.go b/popx/migration_info.go index 0416d8cd..92b9738e 100644 --- a/popx/migration_info.go +++ b/popx/migration_info.go @@ -4,9 +4,10 @@ package popx import ( - "fmt" "sort" + "github.com/pkg/errors" + "github.com/gobuffalo/pop/v6" ) @@ -18,23 +19,28 @@ type Migration struct { Version string // Name of the migration (create_widgets) Name string - // Direction of the migration (up) + // Direction of the migration (up|down) Direction string - // Type of migration (sql) + // Type of migration (sql|go) Type string // DB type (all|postgres|mysql...) DBType string - // Runner function to run/execute the migration + // Runner function to run/execute the migration. Will be wrapped in a + // database transaction. Mutually exclusive with RunnerNoTx Runner func(Migration, *pop.Connection, *pop.Tx) error + // RunnerNoTx function to run/execute the migration. NOT wrapped in a + // database transaction. Mutually exclusive with Runner. + RunnerNoTx func(Migration, *pop.Connection) error } -// Run the migration. Returns an error if there is -// no mf.Runner defined. -func (mf Migration) Run(c *pop.Connection, tx *pop.Tx) error { - if mf.Runner == nil { - return fmt.Errorf("no runner defined for %s", mf.Path) +func (m Migration) Valid() error { + if m.Runner == nil && m.RunnerNoTx == nil { + return errors.Errorf("no runner defined for %s", m.Path) + } + if m.Runner != nil && m.RunnerNoTx != nil { + return errors.Errorf("incompatible transaction and non-transaction runners defined for %s", m.Path) } - return mf.Runner(mf, c, tx) + return nil } // Migrations is a collection of Migration diff --git a/popx/migrator.go b/popx/migrator.go index 28eeec59..2748ef50 100644 --- a/popx/migrator.go +++ b/popx/migrator.go @@ -101,18 +101,20 @@ func (m *Migrator) UpTo(ctx context.Context, step int) (applied int, err error) mtn := m.sanitizedMigrationTableName(c) mfs := m.Migrations["up"].SortAndFilter(c.Dialect.Name()) for _, mi := range mfs { + l := m.l.WithField("version", mi.Version).WithField("migration_name", mi.Name).WithField("migration_file", mi.Path) + exists, err := c.Where("version = ?", mi.Version).Exists(mtn) if err != nil { return errors.Wrapf(err, "problem checking for migration version %s", mi.Version) } if exists { - m.l.WithField("version", mi.Version).Debug("Migration has already been applied, skipping.") + l.Debug("Migration has already been applied, skipping.") continue } if len(mi.Version) > 14 { - m.l.WithField("version", mi.Version).Debug("Migration has not been applied but it might be a legacy migration, investigating.") + l.Debug("Migration has not been applied but it might be a legacy migration, investigating.") legacyVersion := mi.Version[:14] exists, err = c.Where("version = ?", legacyVersion).Exists(mtn) @@ -121,7 +123,7 @@ func (m *Migrator) UpTo(ctx context.Context, step int) (applied int, err error) } if exists { - m.l.WithField("version", mi.Version).WithField("legacy_version", legacyVersion).WithField("migration_table", mtn).Debug("Migration has already been applied in a legacy migration run. Updating version in migration table.") + l.WithField("legacy_version", legacyVersion).WithField("migration_table", mtn).Debug("Migration has already been applied in a legacy migration run. Updating version in migration table.") if err := m.isolatedTransaction(ctx, "init-migrate", func(conn *pop.Connection) error { // We do not want to remove the legacy migration version or subsequent migrations might be applied twice. // @@ -141,23 +143,40 @@ func (m *Migrator) UpTo(ctx context.Context, step int) (applied int, err error) } } - m.l.WithField("version", mi.Version).Debug("Migration has not yet been applied, running migration.") + l.Info("Migration has not yet been applied, running migration.") + + if err := mi.Valid(); err != nil { + return err + } + + if mi.Runner != nil { + err := m.isolatedTransaction(ctx, "up", func(conn *pop.Connection) error { + if err := mi.Runner(mi, conn, conn.TX); err != nil { + return err + } - if err = m.isolatedTransaction(ctx, "up", func(conn *pop.Connection) error { - if err := mi.Run(conn, conn.TX); err != nil { + // #nosec G201 - mtn is a system-wide const + if err := conn.RawQuery(fmt.Sprintf("INSERT INTO %s (version) VALUES (?)", mtn), mi.Version).Exec(); err != nil { + return errors.Wrapf(err, "problem inserting migration version %s", mi.Version) + } + return nil + }) + if err != nil { + return err + } + } else { + l.Warn("Migration has requested running outside a transaction. Proceed with caution.") + if err := mi.RunnerNoTx(mi, c); err != nil { return err } // #nosec G201 - mtn is a system-wide const - if _, err = conn.TX.Exec(fmt.Sprintf("INSERT INTO %s (version) VALUES ('%s')", mtn, mi.Version)); err != nil { - return errors.Wrapf(err, "problem inserting migration version %s", mi.Version) + if err := c.RawQuery(fmt.Sprintf("INSERT INTO %s (version) VALUES (?)", mtn), mi.Version).Exec(); err != nil { + return errors.Wrapf(err, "problem inserting migration version %s. YOUR DATABASE MAY BE IN AN INCONSISTENT STATE! MANUAL INTERVENTION REQUIRED!", mi.Version) } - return nil - }); err != nil { - return err } - m.l.Debugf("> %s", mi.Name) + l.Infof("> %s applied successfully", mi.Name) applied++ if step > 0 && applied >= step { break @@ -215,21 +234,37 @@ func (m *Migrator) Down(ctx context.Context, step int) error { return errors.Errorf("migration version %s does not exist", mi.Version) } - err = m.isolatedTransaction(ctx, "down", func(conn *pop.Connection) error { - err := mi.Run(conn, conn.TX) + if err := mi.Valid(); err != nil { + return err + } + + if mi.Runner != nil { + err := m.isolatedTransaction(ctx, "down", func(conn *pop.Connection) error { + err := mi.Runner(mi, conn, conn.TX) + if err != nil { + return err + } + + // #nosec G201 - mtn is a system-wide const + if err := conn.RawQuery(fmt.Sprintf("DELETE FROM %s WHERE version = ?", mtn), mi.Version).Exec(); err != nil { + return errors.Wrapf(err, "problem deleting migration version %s", mi.Version) + } + + return nil + }) + if err != nil { + return err + } + } else { + err := mi.RunnerNoTx(mi, c) if err != nil { return err } // #nosec G201 - mtn is a system-wide const - if err = conn.RawQuery(fmt.Sprintf("DELETE FROM %s WHERE version = ?", mtn), mi.Version).Exec(); err != nil { - return errors.Wrapf(err, "problem deleting migration version %s", mi.Version) + if err := c.RawQuery(fmt.Sprintf("DELETE FROM %s WHERE version = ?", mtn), mi.Version).Exec(); err != nil { + return errors.Wrapf(err, "problem deleting migration version %s. YOUR DATABASE MAY BE IN AN INCONSISTENT STATE! MANUAL INTERVENTION REQUIRED!", mi.Version) } - - return nil - }) - if err != nil { - return err } m.l.Debugf("< %s", mi.Name) @@ -512,13 +547,6 @@ func (m *Migrator) DumpMigrationSchema(ctx context.Context) error { return nil } -func (m *Migrator) wrapSpan(ctx context.Context, opName string, f func(ctx context.Context, span trace.Span) error) error { - span, ctx := m.startSpan(ctx, opName) - defer span.End() - - return f(ctx, span) -} - func (m *Migrator) startSpan(ctx context.Context, opName string) (trace.Span, context.Context) { tracer := otel.Tracer(tracingComponent) if m.tracer.IsLoaded() { From 126d8c9099b5377872019d9cdbd90cb7fa7528a8 Mon Sep 17 00:00:00 2001 From: Henning Perl Date: Thu, 29 Jun 2023 11:57:35 +0200 Subject: [PATCH 04/56] chore(deps): update go-jose (#700) go-jose v2 is deprecated, and development moved to https://github.com/go-jose/go-jose, v3. --- go.mod | 6 ------ go.sum | 3 +-- josex/generate.go | 2 +- josex/public.go | 2 +- josex/utils.go | 2 +- jwksx/fetcher.go | 2 +- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index e91a58e1..ca98c552 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,6 @@ module github.com/ory/x go 1.19 -replace ( - github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt v3.2.2+incompatible // https://github.com/dgrijalva/jwt-go/issues/482 - github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2 // https://github.com/advisories/GHSA-c3h9-896r-86jm -) - require ( github.com/avast/retry-go/v4 v4.3.0 github.com/bmatcuk/doublestar/v2 v2.0.4 @@ -95,7 +90,6 @@ require ( gonum.org/v1/plot v0.12.0 google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.1 - gopkg.in/square/go-jose.v2 v2.6.0 ) require ( diff --git a/go.sum b/go.sum index 1308c12d..8309b983 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,7 @@ github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70d github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -1410,8 +1411,6 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/josex/generate.go b/josex/generate.go index c8185d8a..c2373e97 100644 --- a/josex/generate.go +++ b/josex/generate.go @@ -26,7 +26,7 @@ import ( "errors" "fmt" - "gopkg.in/square/go-jose.v2" + "github.com/square/go-jose/v3" ) // NewSigningKey generates a keypair for corresponding SignatureAlgorithm. diff --git a/josex/public.go b/josex/public.go index 562e2478..4d7baf92 100644 --- a/josex/public.go +++ b/josex/public.go @@ -3,7 +3,7 @@ package josex import ( "crypto" - "gopkg.in/square/go-jose.v2" + "github.com/square/go-jose/v3" ) // ToPublicKey returns the public key of the given private key. diff --git a/josex/utils.go b/josex/utils.go index df18e030..fa0fc408 100644 --- a/josex/utils.go +++ b/josex/utils.go @@ -22,7 +22,7 @@ import ( "errors" "fmt" - "gopkg.in/square/go-jose.v2" + "github.com/square/go-jose/v3" ) // LoadJSONWebKey returns a *jose.JSONWebKey for a given JSON string. diff --git a/jwksx/fetcher.go b/jwksx/fetcher.go index fe7d49a3..8743f032 100644 --- a/jwksx/fetcher.go +++ b/jwksx/fetcher.go @@ -9,7 +9,7 @@ import ( "sync" "github.com/pkg/errors" - "gopkg.in/square/go-jose.v2" + "github.com/square/go-jose/v3" ) // Fetcher is a small helper for fetching JSON Web Keys from remote endpoints. From d358437a3c71cbf93fc88e0b83b90e8973808d3b Mon Sep 17 00:00:00 2001 From: Henning Perl Date: Thu, 29 Jun 2023 17:20:20 +0200 Subject: [PATCH 05/56] chore(deps): update go-jose (#701) go-jose v2 is deprecated, and development moved to https://github.com/go-jose/go-jose, v3. --- go.mod | 4 ++-- go.sum | 10 ++++++---- josex/generate.go | 2 +- josex/public.go | 2 +- josex/utils.go | 2 +- jwksx/fetcher.go | 2 +- jwksx/generator.go | 5 ++--- jwksx/generator_test.go | 2 +- modx/version_test.go | 2 +- 9 files changed, 16 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index ca98c552..ee968b8f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ory/x -go 1.19 +go 1.20 require ( github.com/avast/retry-go/v4 v4.3.0 @@ -15,6 +15,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/ghodss/yaml v1.0.0 github.com/go-bindata/go-bindata v3.1.2+incompatible + github.com/go-jose/go-jose/v3 v3.0.0 github.com/go-openapi/jsonpointer v0.19.5 github.com/go-openapi/runtime v0.24.2 github.com/go-sql-driver/mysql v1.7.0 @@ -63,7 +64,6 @@ require ( github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/square/go-jose/v3 v3.0.0-20200630053402-0a67ce9b0693 github.com/stretchr/testify v1.8.1 github.com/tidwall/gjson v1.14.3 github.com/tidwall/pretty v1.2.1 diff --git a/go.sum b/go.sum index 8309b983..0ee7d941 100644 --- a/go.sum +++ b/go.sum @@ -166,7 +166,6 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= @@ -202,6 +201,8 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -325,9 +326,10 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc= github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -551,6 +553,7 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -835,8 +838,6 @@ github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/y github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= -github.com/square/go-jose/v3 v3.0.0-20200630053402-0a67ce9b0693 h1:wD1IWQwAhdWclCwaf6DdzgCAe9Bfz1M+4AHRd7N786Y= -github.com/square/go-jose/v3 v3.0.0-20200630053402-0a67ce9b0693/go.mod h1:6hSY48PjDm4UObWmGLyJE9DxYVKTgR9kbCspXXJEhcU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -1210,6 +1211,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/josex/generate.go b/josex/generate.go index c2373e97..055a4c5c 100644 --- a/josex/generate.go +++ b/josex/generate.go @@ -26,7 +26,7 @@ import ( "errors" "fmt" - "github.com/square/go-jose/v3" + "github.com/go-jose/go-jose/v3" ) // NewSigningKey generates a keypair for corresponding SignatureAlgorithm. diff --git a/josex/public.go b/josex/public.go index 4d7baf92..667a2cbb 100644 --- a/josex/public.go +++ b/josex/public.go @@ -3,7 +3,7 @@ package josex import ( "crypto" - "github.com/square/go-jose/v3" + "github.com/go-jose/go-jose/v3" ) // ToPublicKey returns the public key of the given private key. diff --git a/josex/utils.go b/josex/utils.go index fa0fc408..036f36aa 100644 --- a/josex/utils.go +++ b/josex/utils.go @@ -22,7 +22,7 @@ import ( "errors" "fmt" - "github.com/square/go-jose/v3" + "github.com/go-jose/go-jose/v3" ) // LoadJSONWebKey returns a *jose.JSONWebKey for a given JSON string. diff --git a/jwksx/fetcher.go b/jwksx/fetcher.go index 8743f032..01958e1d 100644 --- a/jwksx/fetcher.go +++ b/jwksx/fetcher.go @@ -8,8 +8,8 @@ import ( "net/http" "sync" + "github.com/go-jose/go-jose/v3" "github.com/pkg/errors" - "github.com/square/go-jose/v3" ) // Fetcher is a small helper for fetching JSON Web Keys from remote endpoints. diff --git a/jwksx/generator.go b/jwksx/generator.go index 576a3d36..7fabb117 100644 --- a/jwksx/generator.go +++ b/jwksx/generator.go @@ -12,10 +12,9 @@ import ( "crypto/x509" "io" - "github.com/pkg/errors" - + "github.com/go-jose/go-jose/v3" "github.com/gofrs/uuid" - "github.com/square/go-jose/v3" + "github.com/pkg/errors" "golang.org/x/crypto/ed25519" ) diff --git a/jwksx/generator_test.go b/jwksx/generator_test.go index 2516446f..2f0c65eb 100644 --- a/jwksx/generator_test.go +++ b/jwksx/generator_test.go @@ -7,7 +7,7 @@ import ( "fmt" "testing" - "github.com/square/go-jose/v3" + "github.com/go-jose/go-jose/v3" "github.com/stretchr/testify/require" ) diff --git a/modx/version_test.go b/modx/version_test.go index 13e0f7c8..1ea2bf7f 100644 --- a/modx/version_test.go +++ b/modx/version_test.go @@ -68,7 +68,7 @@ require ( github.com/spf13/cast v1.3.2-0.20200723214538-8d17101741c8 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 - github.com/square/go-jose/v3 v3.0.0-20200630053402-0a67ce9b0693 + github.com/go-jose/go-jose/v3 v3.0.0-20200630053402-0a67ce9b0693 github.com/stretchr/testify v1.6.1 github.com/tidwall/gjson v1.3.2 github.com/tidwall/sjson v1.0.4 From 6314ad0470b2a224f15b8ed032ab50d6e8c310ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:06:06 +0200 Subject: [PATCH 06/56] chore(deps): bump google.golang.org/grpc from 1.50.1 to 1.53.0 (#702) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.50.1 to 1.53.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.50.1...v1.53.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 7 +++---- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index ee968b8f..b5b23b2d 100644 --- a/go.mod +++ b/go.mod @@ -88,7 +88,7 @@ require ( golang.org/x/net v0.7.0 golang.org/x/sync v0.1.0 gonum.org/v1/plot v0.12.0 - google.golang.org/grpc v1.50.1 + google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.28.1 ) @@ -102,7 +102,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/cli v20.10.21+incompatible // indirect @@ -137,7 +137,6 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/pprof v0.0.0-20221010195024-131d412537ea // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.12.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -197,7 +196,7 @@ require ( golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.2.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 0ee7d941..a5a35c36 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,9 @@ github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -407,7 +408,6 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -1355,8 +1355,8 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 h1:GEgb2jF5zxsFJpJfg9RoDDWm7tiwc/DDSTE2BtLUkXU= -google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1378,8 +1378,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= -google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99 h1:qA8rMbz1wQ4DOFfM2ouD29DG9aHWBm6ZOy9BGxiUMmY= google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From adf235e997f6c73c811d8cfbdf3e56a0bde43652 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Mon, 17 Jul 2023 09:04:19 +0200 Subject: [PATCH 07/56] fix: remove up-to-step event (#703) --- popx/migrator.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/popx/migrator.go b/popx/migrator.go index 2748ef50..b388ec23 100644 --- a/popx/migrator.go +++ b/popx/migrator.go @@ -92,9 +92,6 @@ func (m *Migrator) Up(ctx context.Context) error { func (m *Migrator) UpTo(ctx context.Context, step int) (applied int, err error) { span, ctx := m.startSpan(ctx, MigrationUpOpName) defer span.End() - span.AddEvent("up_to_step", trace.WithAttributes( - attribute.Int("up_to_step", step), - )) c := m.Connection.WithContext(ctx) err = m.exec(ctx, func() error { From 4c979b650f3ab0d4b62b77b39cb45b79ce336cd1 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Tue, 18 Jul 2023 09:02:14 +0200 Subject: [PATCH 08/56] fix: use incoming TLS to detect originalScheme (#705) --- proxy/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index f331c1a2..54ec349b 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -82,7 +82,7 @@ func rewriter(o *options) func(*httputil.ProxyRequest) { if forwardedProto := r.In.Header.Get("X-Forwarded-Proto"); forwardedProto != "" { c.originalScheme = forwardedProto - } else if r.Out.TLS == nil { + } else if r.In.TLS == nil { c.originalScheme = "http" } else { c.originalScheme = "https" From 3b15b7c30ded6ede59cc289a09e20008213eef31 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Tue, 18 Jul 2023 10:18:13 +0200 Subject: [PATCH 09/56] feat: support override for originalScheme (#704) --- proxy/proxy.go | 36 ++++++++++++++++++++++++------------ proxy/rewrites_test.go | 15 +++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index 54ec349b..15492f86 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -58,6 +58,8 @@ type ( // originalScheme is the original scheme of the request. // This value will be maintained internally by the proxy. originalScheme string + // ForceOriginalSchemeHTTP forces the original scheme to be https if enabled. + ForceOriginalSchemeHTTPS bool } Options func(*options) contextKey string @@ -67,6 +69,26 @@ const ( hostConfigKey contextKey = "host config" ) +func (c *HostConfig) setScheme(r *httputil.ProxyRequest) { + if c.ForceOriginalSchemeHTTPS { + c.originalScheme = "https" + } else if forwardedProto := r.In.Header.Get("X-Forwarded-Proto"); forwardedProto != "" { + c.originalScheme = forwardedProto + } else if r.In.TLS == nil { + c.originalScheme = "http" + } else { + c.originalScheme = "https" + } +} + +func (c *HostConfig) setHost(r *httputil.ProxyRequest) { + if forwardedHost := r.In.Header.Get("X-Forwarded-Host"); forwardedHost != "" { + c.originalHost = forwardedHost + } else { + c.originalHost = r.In.Host + } +} + // rewriter is a custom internal function for altering a http.Request func rewriter(o *options) func(*httputil.ProxyRequest) { return func(r *httputil.ProxyRequest) { @@ -80,18 +102,8 @@ func rewriter(o *options) func(*httputil.ProxyRequest) { return } - if forwardedProto := r.In.Header.Get("X-Forwarded-Proto"); forwardedProto != "" { - c.originalScheme = forwardedProto - } else if r.In.TLS == nil { - c.originalScheme = "http" - } else { - c.originalScheme = "https" - } - if forwardedHost := r.In.Header.Get("X-Forwarded-Host"); forwardedHost != "" { - c.originalHost = forwardedHost - } else { - c.originalHost = r.In.Host - } + c.setScheme(r) + c.setHost(r) *r.Out = *r.Out.WithContext(context.WithValue(ctx, hostConfigKey, c)) headerRequestRewrite(r.Out, c) diff --git a/proxy/rewrites_test.go b/proxy/rewrites_test.go index 8c5e67b1..0dc776cb 100644 --- a/proxy/rewrites_test.go +++ b/proxy/rewrites_test.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "net/http/httputil" "net/url" "strings" "testing" @@ -50,6 +51,19 @@ func TestRewrites(t *testing.T) { assert.Equal(t, "/bar", req.URL.Path) }) + t.Run("suite=HTTPS override", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "http://example.com/foo/bar", nil) + require.NoError(t, err) + + c := &HostConfig{} + c.setScheme(&httputil.ProxyRequest{In: req, Out: &http.Request{}}) + assert.Equal(t, "http", c.originalScheme) + + c.ForceOriginalSchemeHTTPS = true + c.setScheme(&httputil.ProxyRequest{In: req, Out: &http.Request{}}) + assert.Equal(t, "https", c.originalScheme) + }) + t.Run("suit=HeaderResponse", func(t *testing.T) { newOKResp := func(cookie, location string) *http.Response { header := http.Header{} @@ -68,6 +82,7 @@ func TestRewrites(t *testing.T) { ContentLength: 0, } } + t.Run("case=replace location and cookie", func(t *testing.T) { upstreamHost := "some-project-1234.oryapis.com" From e77f73f1a6e280a6a2669d02e39845e4201eae0e Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Tue, 18 Jul 2023 12:13:19 +0200 Subject: [PATCH 10/56] feat(proxy): flag to keep forwarded headers (#706) --- jsonx/patch.go | 2 +- proxy/proxy.go | 46 +++++++++++++++++++++++++++------------- proxy/proxy_full_test.go | 23 ++++++++++---------- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/jsonx/patch.go b/jsonx/patch.go index 60f36f13..7b732e4a 100644 --- a/jsonx/patch.go +++ b/jsonx/patch.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/evanphx/json-patch/v5" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/ory/x/pointerx" ) diff --git a/proxy/proxy.go b/proxy/proxy.go index 15492f86..42c25735 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -17,8 +17,8 @@ import ( type ( RespMiddleware func(resp *http.Response, config *HostConfig, body []byte) ([]byte, error) - ReqMiddleware func(req *http.Request, config *HostConfig, body []byte) ([]byte, error) - HostMapper func(ctx context.Context, r *http.Request) (*HostConfig, error) + ReqMiddleware func(req *httputil.ProxyRequest, config *HostConfig, body []byte) ([]byte, error) + HostMapper func(ctx context.Context, r *http.Request) (context.Context, *HostConfig, error) options struct { hostMapper HostMapper onResError func(*http.Response, error) error @@ -52,6 +52,9 @@ type ( // PathPrefix is a prefix that is prepended on the original host, // but removed before forwarding. PathPrefix string + // TrustForwardedHosts is a flag that indicates whether the proxy should trust the + // X-Forwarded-* headers or not. + TrustForwardedHeaders bool // originalHost the original hostname the request is coming from. // This value will be maintained internally by the proxy. originalHost string @@ -96,16 +99,28 @@ func rewriter(o *options) func(*httputil.ProxyRequest) { ctx, span := otel.GetTracerProvider().Tracer("").Start(ctx, "x.proxy") defer span.End() - c, err := o.getHostConfig(r.Out) + ctx, c, err := o.getHostConfig(ctx, r.In) if err != nil { o.onReqError(r.Out, err) return } + if c.TrustForwardedHeaders { + headers := []string{ + "X-Forwarded-Host", + "X-Forwarded-Proto", + "X-Forwarded-For", + } + for _, h := range headers { + if v := r.In.Header.Get(h); v != "" { + r.Out.Header.Set(h, v) + } + } + } + c.setScheme(r) c.setHost(r) - *r.Out = *r.Out.WithContext(context.WithValue(ctx, hostConfigKey, c)) headerRequestRewrite(r.Out, c) var body []byte @@ -120,7 +135,7 @@ func rewriter(o *options) func(*httputil.ProxyRequest) { } for _, m := range o.reqMiddlewares { - if body, err = m(r.Out, c, body); err != nil { + if body, err = m(r, c, body); err != nil { o.onReqError(r.Out, err) return } @@ -141,7 +156,7 @@ func rewriter(o *options) func(*httputil.ProxyRequest) { // modifyResponse is a custom internal function for altering a http.Response func modifyResponse(o *options) func(*http.Response) error { return func(r *http.Response) error { - c, err := o.getHostConfig(r.Request) + _, c, err := o.getHostConfig(r.Request.Context(), r.Request) if err != nil { return err } @@ -209,24 +224,24 @@ func WithErrorHandler(eh func(w http.ResponseWriter, r *http.Request, err error) } } -func (o *options) getHostConfig(r *http.Request) (*HostConfig, error) { - if cached, ok := r.Context().Value(hostConfigKey).(*HostConfig); ok && cached != nil { - return cached, nil +func (o *options) getHostConfig(ctx context.Context, r *http.Request) (context.Context, *HostConfig, error) { + if cached, ok := ctx.Value(hostConfigKey).(*HostConfig); ok && cached != nil { + return ctx, cached, nil } - c, err := o.hostMapper(r.Context(), r) + ctx, c, err := o.hostMapper(ctx, r) if err != nil { - return nil, err + return nil, nil, err } // cache the host config in the request context // this will be passed on to the request and response proxy functions - *r = *r.WithContext(context.WithValue(r.Context(), hostConfigKey, c)) - return c, nil + ctx = context.WithValue(ctx, hostConfigKey, c) + return ctx, c, nil } func (o *options) beforeProxyMiddleware(h http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { // get the hostmapper configurations before the request is proxied - c, err := o.getHostConfig(request) + ctx, c, err := o.getHostConfig(request.Context(), request) if err != nil { o.onReqError(request, err) return @@ -237,7 +252,8 @@ func (o *options) beforeProxyMiddleware(h http.Handler) http.Handler { if c.CorsEnabled && c.CorsOptions != nil { cors.New(*c.CorsOptions).HandlerFunc(writer, request) } - h.ServeHTTP(writer, request) + + h.ServeHTTP(writer, request.WithContext(ctx)) }) } diff --git a/proxy/proxy_full_test.go b/proxy/proxy_full_test.go index 74b9b5ae..f11ec79e 100644 --- a/proxy/proxy_full_test.go +++ b/proxy/proxy_full_test.go @@ -102,11 +102,12 @@ func TestFullIntegration(t *testing.T) { onErrorResp := make(chan CustomErrorResp) proxy := httptest.NewTLSServer(New( - func(_ context.Context, r *http.Request) (*HostConfig, error) { - return (<-hostMapper)(r) + func(ctx context.Context, r *http.Request) (context.Context, *HostConfig, error) { + c, err := (<-hostMapper)(r) + return ctx, c, err }, WithTransport(upstreamServer.Client().Transport), - WithReqMiddleware(func(req *http.Request, config *HostConfig, body []byte) ([]byte, error) { + WithReqMiddleware(func(req *httputil.ProxyRequest, config *HostConfig, body []byte) ([]byte, error) { f := <-reqMiddleware if f == nil { return body, nil @@ -268,8 +269,8 @@ func TestFullIntegration(t *testing.T) { assert.Equal(t, "OK", string(body)) assert.Equal(t, "1234", r.Header.Get("Some-Header")) }, - reqMiddleware: func(req *http.Request, config *HostConfig, body []byte) ([]byte, error) { - req.Host = "noauth.example.com" + reqMiddleware: func(req *httputil.ProxyRequest, config *HostConfig, body []byte) ([]byte, error) { + req.Out.Host = "noauth.example.com" return []byte("this is a new body"), nil }, respMiddleware: func(resp *http.Response, config *HostConfig, body []byte) ([]byte, error) { @@ -538,8 +539,8 @@ func TestBetweenReverseProxies(t *testing.T) { revProxyHandler := httputil.NewSingleHostReverseProxy(urlx.ParseOrPanic(target.URL)) revProxy := httptest.NewServer(revProxyHandler) - thisProxy := httptest.NewServer(New(func(ctx context.Context, _ *http.Request) (*HostConfig, error) { - return &HostConfig{ + thisProxy := httptest.NewServer(New(func(ctx context.Context, _ *http.Request) (context.Context, *HostConfig, error) { + return ctx, &HostConfig{ CookieDomain: "sh", UpstreamHost: urlx.ParseOrPanic(revProxy.URL).Host, UpstreamScheme: urlx.ParseOrPanic(revProxy.URL).Scheme, @@ -635,8 +636,8 @@ func TestProxyProtoMix(t *testing.T) { upstream.Transport = targetServer.Client().Transport upstreamServer := upstreamServerFunc(upstream) - proxy := httptest.NewServer(New(func(ctx context.Context, r *http.Request) (*HostConfig, error) { - return &HostConfig{ + proxy := httptest.NewServer(New(func(ctx context.Context, r *http.Request) (context.Context, *HostConfig, error) { + return ctx, &HostConfig{ CookieDomain: exposedHost, UpstreamHost: urlx.ParseOrPanic(upstreamServer.URL).Host, UpstreamScheme: urlx.ParseOrPanic(upstreamServer.URL).Scheme, @@ -760,8 +761,8 @@ func TestProxyWebsocketRequests(t *testing.T) { } setupProxy := func(targetServer *httptest.Server) *httptest.Server { - proxy := httptest.NewServer(New(func(ctx context.Context, r *http.Request) (*HostConfig, error) { - return &HostConfig{ + proxy := httptest.NewServer(New(func(ctx context.Context, r *http.Request) (context.Context, *HostConfig, error) { + return ctx, &HostConfig{ UpstreamHost: urlx.ParseOrPanic(targetServer.URL).Host, UpstreamScheme: urlx.ParseOrPanic(targetServer.URL).Scheme, TargetHost: urlx.ParseOrPanic(targetServer.URL).Host, From d50f3ce3b4dabde485cbe4bbf86ffb955493e7ce Mon Sep 17 00:00:00 2001 From: Patrik Date: Wed, 26 Jul 2023 12:46:45 +0200 Subject: [PATCH 11/56] feat(clidoc): enable templating for command fields (#707) --- clidoc/generate_test.go | 2 +- clidoc/md_docs.go | 16 ++++++++++++++-- clidoc/testdata/root-child1.md | 6 ++++++ cmdx/usage.go | 28 +++++++++++++++------------- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/clidoc/generate_test.go b/clidoc/generate_test.go index 5ca9e8e1..fc9e069b 100644 --- a/clidoc/generate_test.go +++ b/clidoc/generate_test.go @@ -24,7 +24,7 @@ root child1 <[some argument]> -`} +`, Example: "{{ .CommandPath }} --whatever"} child2 = &cobra.Command{Use: "child2", Run: noopRun, Long: `A sample text child2 diff --git a/clidoc/md_docs.go b/clidoc/md_docs.go index 5bdf8a04..e5131159 100644 --- a/clidoc/md_docs.go +++ b/clidoc/md_docs.go @@ -23,6 +23,8 @@ import ( "sort" "strings" + "github.com/ory/x/cmdx" + "github.com/spf13/cobra" ) @@ -62,7 +64,12 @@ func GenMarkdownCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) buf.WriteString(cmd.Short + "\n\n") if len(cmd.Long) > 0 { buf.WriteString("### Synopsis\n\n") - buf.WriteString(cmd.Long + "\n\n") + long, err := cmdx.TemplateCommandField(cmd, cmd.Long) + if err != nil { + buf.WriteString(fmt.Sprintf("\n\n", err.Error())) + long = cmd.Long + } + buf.WriteString(long + "\n\n") } if cmd.Runnable() { @@ -71,7 +78,12 @@ func GenMarkdownCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) if len(cmd.Example) > 0 { buf.WriteString("### Examples\n\n") - buf.WriteString(fmt.Sprintf("```\n%s\n```\n\n", cmd.Example)) + example, err := cmdx.TemplateCommandField(cmd, cmd.Example) + if err != nil { + buf.WriteString(fmt.Sprintf("\n\n", err.Error())) + example = cmd.Example + } + buf.WriteString(fmt.Sprintf("```\n%s\n```\n\n", example)) } if err := printOptions(buf, cmd, name); err != nil { diff --git a/clidoc/testdata/root-child1.md b/clidoc/testdata/root-child1.md index 120fe915..f9d907ee 100644 --- a/clidoc/testdata/root-child1.md +++ b/clidoc/testdata/root-child1.md @@ -25,6 +25,12 @@ child1 root child1 [flags] ``` +### Examples + +``` +root child1 --whatever +``` + ### Options ``` diff --git a/cmdx/usage.go b/cmdx/usage.go index cd0ae8f1..08ff8971 100644 --- a/cmdx/usage.go +++ b/cmdx/usage.go @@ -54,25 +54,27 @@ Use "{{.CommandPath}} [command] --help" for more information about a command.{{e // The data for the template is the command itself. Especially useful are `.Root.Name` and `.CommandPath`. // This will be inherited by all subcommands, so enabling it on the root command is sufficient. func EnableUsageTemplating(cmds ...*cobra.Command) { - cobra.AddTemplateFunc("insertTemplate", func(cmd *cobra.Command, tmpl string) (string, error) { - t := template.New("") - t.Funcs(usageTemplateFuncs) - t, err := t.Parse(tmpl) - if err != nil { - return "", err - } - var out bytes.Buffer - if err := t.Execute(&out, cmd); err != nil { - return "", err - } - return out.String(), nil - }) + cobra.AddTemplateFunc("insertTemplate", TemplateCommandField) for _, cmd := range cmds { cmd.SetHelpTemplate(helpTemplate) cmd.SetUsageTemplate(usageTemplate) } } +func TemplateCommandField(cmd *cobra.Command, field string) (string, error) { + t := template.New("") + t.Funcs(usageTemplateFuncs) + t, err := t.Parse(field) + if err != nil { + return "", err + } + var out bytes.Buffer + if err := t.Execute(&out, cmd); err != nil { + return "", err + } + return out.String(), nil +} + // DisableUsageTemplating resets the commands usage template to the default. // This can be used to undo the effects of EnableUsageTemplating, specifically for a subcommand. func DisableUsageTemplating(cmds ...*cobra.Command) { From d3779fe5b44b5aba8563d3619f647b2bfa906feb Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Wed, 2 Aug 2023 11:51:59 +0200 Subject: [PATCH 12/56] feat: add breaking test for fsx.Merge --- fsx/merge_test.go | 12 +++++++++ go.mod | 27 ++++++++++---------- go.sum | 63 ++++++++++++++++++++++------------------------- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/fsx/merge_test.go b/fsx/merge_test.go index 3adb4e53..673b47aa 100644 --- a/fsx/merge_test.go +++ b/fsx/merge_test.go @@ -7,6 +7,7 @@ import ( "testing" "testing/fstest" + "github.com/laher/mergefs" "github.com/stretchr/testify/assert" ) @@ -22,4 +23,15 @@ func TestMergeFS(t *testing.T) { m := Merge(a, b) assert.NoError(t, fstest.TestFS(m, "a", "b", "dir", "dir/c", "dir/d")) + + x := fstest.MapFS{ + "x": &fstest.MapFile{}, + "dir/y": &fstest.MapFile{}, + } + + m2 := Merge(m, x) + assert.NoError(t, fstest.TestFS(m2, "a", "b", "dir", "dir/c", "dir/d", "dir/y", "x")) + + m2 = mergefs.Merge(mergefs.Merge(a, b), x) + assert.NoError(t, fstest.TestFS(m2, "a", "b", "dir", "dir/c", "dir/d", "dir/y", "x")) } diff --git a/go.mod b/go.mod index b5b23b2d..2727f516 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/ory/analytics-go/v5 v5.0.1 github.com/ory/dockertest/v3 v3.9.1 github.com/ory/go-acc v0.2.9-0.20230103102148-6b1c9a70dbbe - github.com/ory/herodot v0.9.13 + github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88 github.com/ory/jsonschema/v3 v3.0.7 github.com/pelletier/go-toml v1.9.5 github.com/pkg/errors v0.9.1 @@ -62,7 +62,7 @@ require ( github.com/seatgeek/logrus-gelf-formatter v0.0.0-20210414080842-5b05eb8ff761 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cast v1.5.0 - github.com/spf13/cobra v1.6.1 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.1 github.com/tidwall/gjson v1.14.3 @@ -84,12 +84,12 @@ require ( go.opentelemetry.io/proto/otlp v0.18.0 go.uber.org/goleak v1.2.1 golang.org/x/crypto v0.1.0 - golang.org/x/mod v0.6.0 - golang.org/x/net v0.7.0 + golang.org/x/mod v0.10.0 + golang.org/x/net v0.8.0 golang.org/x/sync v0.1.0 gonum.org/v1/plot v0.12.0 - google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.28.1 + google.golang.org/grpc v1.54.0 + google.golang.org/protobuf v1.30.0 ) require ( @@ -134,7 +134,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/glog v1.0.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20221010195024-131d412537ea // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.0 // indirect @@ -152,6 +152,7 @@ require ( github.com/joho/godotenv v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/laher/mergefs v0.1.1 github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -177,11 +178,11 @@ require ( github.com/sergi/go-diff v1.2.0 // indirect github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e // indirect - github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/afero v1.9.5 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.14.0 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/subosito/gotenv v1.4.1 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -191,12 +192,12 @@ require ( go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect go.opentelemetry.io/otel/metric v0.33.0 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.2.0 // indirect + golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index a5a35c36..ab51d274 100644 --- a/go.sum +++ b/go.sum @@ -364,8 +364,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -464,7 +465,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1: github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= @@ -586,6 +586,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/laher/mergefs v0.1.1 h1:nV2bTS57vrmbMxeR6uvJpI8LyGl3QHj4bLBZO3aUV58= +github.com/laher/mergefs v0.1.1/go.mod h1:FSY1hYy94on4Tz60waRMGdO1awwS23BacqJlqf9lJ9Q= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -611,6 +613,7 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI github.com/markbates/pkger v0.17.1 h1:/MKEtWqtc0mZvu9OinB9UzVN9iYCwLWuyUv4Bw+PCno= github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -694,8 +697,8 @@ github.com/ory/dockertest/v3 v3.9.1/go.mod h1:42Ir9hmvaAPm0Mgibk6mBPi7SFvTXxEcnz github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= github.com/ory/go-acc v0.2.9-0.20230103102148-6b1c9a70dbbe h1:rvu4obdvqR0fkSIJ8IfgzKOWwZ5kOT2UNfLq81Qk7rc= github.com/ory/go-acc v0.2.9-0.20230103102148-6b1c9a70dbbe/go.mod h1:z4n3u6as84LbV4YmgjHhnwtccQqzf4cZlSk9f1FhygI= -github.com/ory/herodot v0.9.13 h1:cN/Z4eOkErl/9W7hDIDLb79IO/bfsH+8yscBjRpB4IU= -github.com/ory/herodot v0.9.13/go.mod h1:IWDs9kSvFQqw/cQ8zi5ksyYvITiUU4dI7glUrhZcJYo= +github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88 h1:J0CIFKdpUeqKbVMw7pQ1qLtUnflRM1JWAcOEq7Hp4yg= +github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88/go.mod h1:MMNmY6MG1uB6fnXYFaHoqdV23DTWctlPsmRCeq/2+wc= github.com/ory/jsonschema/v3 v3.0.7 h1:GQ9qfZDiJqs4l2d3p56dozCChvejQFZyLKGHYzDzOSo= github.com/ory/jsonschema/v3 v3.0.7/go.mod h1:g8c8YOtN4TrR2wYeMdT02GDmzJDI0fEW2nI26BECafY= github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= @@ -711,7 +714,6 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= @@ -815,8 +817,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.5.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= -github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.2-0.20200723214538-8d17101741c8/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -826,8 +828,8 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -855,8 +857,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= @@ -898,7 +900,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -977,7 +978,6 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1028,8 +1028,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1071,15 +1071,14 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1172,7 +1171,6 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1189,8 +1187,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1204,8 +1202,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1274,10 +1273,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1344,7 +1342,6 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1353,10 +1350,9 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1378,10 +1374,9 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99 h1:qA8rMbz1wQ4DOFfM2ouD29DG9aHWBm6ZOy9BGxiUMmY= -google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1395,8 +1390,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 9f09e6132587e1eaed7e3a4a7813d78c674a9548 Mon Sep 17 00:00:00 2001 From: Patrik Date: Wed, 2 Aug 2023 16:27:11 +0200 Subject: [PATCH 13/56] fix(fsx): multiple merges edge cases --- fsx/merge.go | 62 +++++++++++++++++++++++++++-------------------- fsx/merge_test.go | 46 +++++++++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 34 deletions(-) diff --git a/fsx/merge.go b/fsx/merge.go index 1342cf4c..35df2166 100644 --- a/fsx/merge.go +++ b/fsx/merge.go @@ -98,7 +98,7 @@ func (m mergedFS) ReadDir(name string) ([]fs.DirEntry, error) { } entries = append(entries, e...) } - if entries == nil { + if len(entries) == 0 { return nil, errors.WithStack(fs.ErrNotExist) } @@ -147,11 +147,16 @@ func (d *dirEntries) clean() { for i := 1; i < len(*d); i++ { if (*d)[i-1].Name() == (*d)[i].Name() { - if len(*d)-i >= 2 { - *d = append((*d)[:i+1], (*d)[i+2:]...) - } else { - *d = (*d)[:len(*d)-1] + if len(*d)-i == 1 { + // remove the last entry; we're done + *d = (*d)[:i] + return } + // remove the duplicate entry at index i + *d = append((*d)[:i], (*d)[i+1:]...) + + // need to check the same index again + i-- } } } @@ -177,28 +182,31 @@ func (m *mergedFile) Close() error { } func (m *mergedFile) ReadDir(n int) ([]fs.DirEntry, error) { - entries := m.unprocessedDirEntries - - if len(entries) < n || n <= 0 { - allEOF := true - for _, f := range m.files { - if f, ok := f.(fs.ReadDirFile); ok { - e, err := f.ReadDir(n) - switch { - case !errors.Is(err, io.EOF): - allEOF = false - case errors.Is(err, fs.ErrNotExist), errors.Is(err, io.EOF): - case err != nil: - return nil, err - } - entries = append(entries, e...) - } + if m.unprocessedDirEntries != nil { + if n <= 0 { + entries := m.unprocessedDirEntries + m.unprocessedDirEntries = nil + return entries, nil } - if allEOF { - if n > 0 { - return entries, io.EOF + if n >= len(m.unprocessedDirEntries) { + entries := m.unprocessedDirEntries + m.unprocessedDirEntries = nil + return entries, io.EOF + } + + var entries dirEntries + entries, m.unprocessedDirEntries = m.unprocessedDirEntries[:n], m.unprocessedDirEntries[n:] + return entries, nil + } + + var entries dirEntries + for _, f := range m.files { + if f, ok := f.(fs.ReadDirFile); ok { + e, err := f.ReadDir(-1) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err } - return entries, nil + entries = append(entries, e...) } } if entries == nil { @@ -216,6 +224,8 @@ func (m *mergedFile) ReadDir(n int) ([]fs.DirEntry, error) { return entries, io.EOF } - entries, m.unprocessedDirEntries = entries[:n], entries[n:] + if n <= len(entries) && n > 0 { + entries, m.unprocessedDirEntries = entries[:n], entries[n:] + } return entries, nil } diff --git a/fsx/merge_test.go b/fsx/merge_test.go index 673b47aa..b7bd8d4b 100644 --- a/fsx/merge_test.go +++ b/fsx/merge_test.go @@ -20,18 +20,48 @@ func TestMergeFS(t *testing.T) { "b": &fstest.MapFile{}, "dir/d": &fstest.MapFile{}, } - m := Merge(a, b) - - assert.NoError(t, fstest.TestFS(m, "a", "b", "dir", "dir/c", "dir/d")) - x := fstest.MapFS{ "x": &fstest.MapFile{}, "dir/y": &fstest.MapFile{}, } - m2 := Merge(m, x) - assert.NoError(t, fstest.TestFS(m2, "a", "b", "dir", "dir/c", "dir/d", "dir/y", "x")) + assert.NoError(t, fstest.TestFS( + Merge(a, b), + "a", + "b", + "dir", + "dir/c", + "dir/d", + )) + + assert.NoError(t, fstest.TestFS( + Merge(a, b, x), + "a", + "b", + "dir", + "dir/c", + "dir/d", + "dir/y", + "x", + )) + + assert.Error(t, fstest.TestFS( + mergefs.Merge(a, b), + "a", + "b", + "dir", + "dir/c", + "dir/d", + )) - m2 = mergefs.Merge(mergefs.Merge(a, b), x) - assert.NoError(t, fstest.TestFS(m2, "a", "b", "dir", "dir/c", "dir/d", "dir/y", "x")) + assert.Error(t, fstest.TestFS( + mergefs.Merge(mergefs.Merge(a, b), x), + "a", + "b", + "dir", + "dir/c", + "dir/d", + "dir/y", + "x", + )) } From 19829a931daa7e60675daa8d53efedb13051959f Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Thu, 3 Aug 2023 10:05:11 +0200 Subject: [PATCH 14/56] feat: add more tests to fsx --- fsx/merge_test.go | 66 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/fsx/merge_test.go b/fsx/merge_test.go index b7bd8d4b..f422dccd 100644 --- a/fsx/merge_test.go +++ b/fsx/merge_test.go @@ -11,20 +11,22 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMergeFS(t *testing.T) { - a := fstest.MapFS{ +var ( + a = fstest.MapFS{ "a": &fstest.MapFile{}, "dir/c": &fstest.MapFile{}, } - b := fstest.MapFS{ + b = fstest.MapFS{ "b": &fstest.MapFile{}, "dir/d": &fstest.MapFile{}, } - x := fstest.MapFS{ + x = fstest.MapFS{ "x": &fstest.MapFile{}, "dir/y": &fstest.MapFile{}, } +) +func TestMergeFS(t *testing.T) { assert.NoError(t, fstest.TestFS( Merge(a, b), "a", @@ -44,7 +46,39 @@ func TestMergeFS(t *testing.T) { "dir/y", "x", )) + assert.NoError(t, fstest.TestFS( + Merge(x, b, a), + "a", + "b", + "dir", + "dir/c", + "dir/d", + "dir/y", + "x", + )) + assert.NoError(t, fstest.TestFS( + Merge(Merge(a, b), x), + "a", + "b", + "dir", + "dir/c", + "dir/d", + "dir/y", + "x", + )) + assert.NoError(t, fstest.TestFS( + Merge(Merge(x, b), a), + "a", + "b", + "dir", + "dir/c", + "dir/d", + "dir/y", + "x", + )) +} +func TestLaherMergeFS(t *testing.T) { assert.Error(t, fstest.TestFS( mergefs.Merge(a, b), "a", @@ -54,7 +88,9 @@ func TestMergeFS(t *testing.T) { "dir/d", )) - assert.Error(t, fstest.TestFS( + t.Skip("laher/mergefs does not handle recursive merges correctly") + + assert.NoError(t, fstest.TestFS( mergefs.Merge(mergefs.Merge(a, b), x), "a", "b", @@ -64,4 +100,24 @@ func TestMergeFS(t *testing.T) { "dir/y", "x", )) + assert.NoError(t, fstest.TestFS( + mergefs.Merge(a, mergefs.Merge(b, x)), + "a", + "b", + "dir", + "dir/c", + "dir/d", + "dir/y", + "x", + )) + assert.NoError(t, fstest.TestFS( + mergefs.Merge(x, mergefs.Merge(b, a)), + "a", + "b", + "dir", + "dir/c", + "dir/d", + "dir/y", + "x", + )) } From 21b16390046723a607acb8b29e57dc3f09b54b4f Mon Sep 17 00:00:00 2001 From: Sergey Plaunov Date: Thu, 3 Aug 2023 13:19:12 +0300 Subject: [PATCH 15/56] feat: add `Authorization` header to OTEL config (CORE-2291) (#708) --- configx/provider.go | 1 + otelx/config.go | 7 ++++--- otelx/config.schema.json | 4 ++++ otelx/otlp.go | 6 ++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/configx/provider.go b/configx/provider.go index 6f623c8f..6ae4c472 100644 --- a/configx/provider.go +++ b/configx/provider.go @@ -454,6 +454,7 @@ func (p *Provider) TracingConfig(serviceName string) *otelx.Config { Sampling: otelx.OTLPSampling{ SamplingRatio: p.Float64("tracing.providers.otlp.sampling.sampling_ratio"), }, + AuthorizationHeader: p.String("tracing.providers.otlp.authorization_header"), }, }, } diff --git a/otelx/config.go b/otelx/config.go index 66b033dd..32c2ec56 100644 --- a/otelx/config.go +++ b/otelx/config.go @@ -20,9 +20,10 @@ type ZipkinConfig struct { } type OTLPConfig struct { - ServerURL string `json:"server_url"` - Insecure bool `json:"insecure"` - Sampling OTLPSampling `json:"sampling"` + ServerURL string `json:"server_url"` + Insecure bool `json:"insecure"` + Sampling OTLPSampling `json:"sampling"` + AuthorizationHeader string `json:"authorization_header"` } type JaegerSampling struct { diff --git a/otelx/config.schema.json b/otelx/config.schema.json index dff7d119..2593b89d 100644 --- a/otelx/config.schema.json +++ b/otelx/config.schema.json @@ -134,6 +134,10 @@ "examples": [0.4] } } + }, + "authorization_header": { + "type": "string", + "examples": ["Bearer 2389s8fs9d8fus9f"] } } } diff --git a/otelx/otlp.go b/otelx/otlp.go index 8fdc40d1..e05db7c7 100644 --- a/otelx/otlp.go +++ b/otelx/otlp.go @@ -29,6 +29,12 @@ func SetupOTLP(t *Tracer, tracerName string, c *Config) (trace.Tracer, error) { clientOpts = append(clientOpts, otlptracehttp.WithInsecure()) } + if c.Providers.OTLP.AuthorizationHeader != "" { + clientOpts = append(clientOpts, + otlptracehttp.WithHeaders(map[string]string{"Authorization": c.Providers.OTLP.AuthorizationHeader}), + ) + } + exp, err := otlptrace.New( ctx, otlptracehttp.NewClient(clientOpts...), ) From e9342cd6033872227564c9dbbc20891f4d3caf83 Mon Sep 17 00:00:00 2001 From: aeneasr <3372410+aeneasr@users.noreply.github.com> Date: Thu, 3 Aug 2023 15:40:13 +0000 Subject: [PATCH 16/56] chore: update repository templates to https://github.com/ory/meta/commit/af28aff50b62a9eeb69de4842e0e164f82c9e066 --- CODE_OF_CONDUCT.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 4861c9d1..9cebaf35 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -39,6 +39,16 @@ Examples of unacceptable behavior include: - Other conduct which could reasonably be considered inappropriate in a professional setting +## Open Source Community Support + +Ory Open source software is collaborative and based on contributions by +developers in the Ory community. There is no obligation from Ory to help with +individual problems. If Ory open source software is used in production in a +for-profit company or enterprise environment, we mandate a paid support contract +where Ory is obligated under their service level agreements (SLAs) to offer a +defined level of availability and responsibility. For more information about +paid support please contact us at sales@ory.sh. + ## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of From 4a2efe9415b0a04ac590280714cc3fae728ba0e1 Mon Sep 17 00:00:00 2001 From: aeneasr <3372410+aeneasr@users.noreply.github.com> Date: Mon, 7 Aug 2023 14:31:16 +0000 Subject: [PATCH 17/56] chore: update repository templates to https://github.com/ory/meta/commit/ac80097fa427e7ae39820c59cac62dc6e11b9aff --- .github/ISSUE_TEMPLATE/BUG-REPORT.yml | 10 ++++++++-- .github/ISSUE_TEMPLATE/DESIGN-DOC.yml | 10 ++++++++-- .github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml | 10 ++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml index 7673ad89..4efddb6e 100644 --- a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml +++ b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml @@ -24,8 +24,6 @@ body: "I have read and am following this repository's [Contribution Guidelines](https://github.com/ory/x/blob/master/CONTRIBUTING.md)." required: true - - label: - "This issue affects my [Ory Network](https://www.ory.sh/) project." - label: "I have joined the [Ory Community Slack](https://slack.ory.sh)." - label: @@ -33,6 +31,14 @@ body: Newsletter](https://ory.us10.list-manage.com/subscribe?u=ffb1a878e4ec6c0ed312a3480&id=f605a41b53)." id: checklist type: checkboxes + - attributes: + description: + "Enter the slug or API URL of the affected Ory Network project. Leave + empty when you are self-hosting." + label: "Ory Network Project" + placeholder: "https://.projects.oryapis.com" + id: ory-network-project + type: input - attributes: description: "A clear and concise description of what the bug is." label: "Describe the bug" diff --git a/.github/ISSUE_TEMPLATE/DESIGN-DOC.yml b/.github/ISSUE_TEMPLATE/DESIGN-DOC.yml index 89048973..3448d55b 100644 --- a/.github/ISSUE_TEMPLATE/DESIGN-DOC.yml +++ b/.github/ISSUE_TEMPLATE/DESIGN-DOC.yml @@ -35,8 +35,6 @@ body: "I have read and am following this repository's [Contribution Guidelines](https://github.com/ory/x/blob/master/CONTRIBUTING.md)." required: true - - label: - "This issue affects my [Ory Network](https://www.ory.sh/) project." - label: "I have joined the [Ory Community Slack](https://slack.ory.sh)." - label: @@ -44,6 +42,14 @@ body: Newsletter](https://ory.us10.list-manage.com/subscribe?u=ffb1a878e4ec6c0ed312a3480&id=f605a41b53)." id: checklist type: checkboxes + - attributes: + description: + "Enter the slug or API URL of the affected Ory Network project. Leave + empty when you are self-hosting." + label: "Ory Network Project" + placeholder: "https://.projects.oryapis.com" + id: ory-network-project + type: input - attributes: description: | This section gives the reader a very rough overview of the landscape in which the new system is being built and what is actually being built. This isn’t a requirements doc. Keep it succinct! The goal is that readers are brought up to speed but some previous knowledge can be assumed and detailed info can be linked to. This section should be entirely focused on objective background facts. diff --git a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml index 287bfe42..d280af6d 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml +++ b/.github/ISSUE_TEMPLATE/FEATURE-REQUEST.yml @@ -28,8 +28,6 @@ body: "I have read and am following this repository's [Contribution Guidelines](https://github.com/ory/x/blob/master/CONTRIBUTING.md)." required: true - - label: - "This issue affects my [Ory Network](https://www.ory.sh/) project." - label: "I have joined the [Ory Community Slack](https://slack.ory.sh)." - label: @@ -37,6 +35,14 @@ body: Newsletter](https://ory.us10.list-manage.com/subscribe?u=ffb1a878e4ec6c0ed312a3480&id=f605a41b53)." id: checklist type: checkboxes + - attributes: + description: + "Enter the slug or API URL of the affected Ory Network project. Leave + empty when you are self-hosting." + label: "Ory Network Project" + placeholder: "https://.projects.oryapis.com" + id: ory-network-project + type: input - attributes: description: "Is your feature request related to a problem? Please describe." From 17527e9413f4c4a760393e55243d1e3b39829813 Mon Sep 17 00:00:00 2001 From: Patrik Date: Thu, 10 Aug 2023 11:21:44 +0200 Subject: [PATCH 18/56] feat: allow exceptions in immutables & CORS hot-reload (#713) --- configx/options.go | 6 ++ configx/provider.go | 32 ++++++++-- configx/provider_watch_test.go | 25 ++++++++ corsx/check_origin.go | 54 ++++++++++++++++ corsx/check_origin_test.go | 111 +++++++++++++++++++++++++++++++++ corsx/middleware.go | 2 + 6 files changed, 225 insertions(+), 5 deletions(-) create mode 100644 corsx/check_origin.go create mode 100644 corsx/check_origin_test.go diff --git a/configx/options.go b/configx/options.go index 1caa763e..5610ee13 100644 --- a/configx/options.go +++ b/configx/options.go @@ -44,6 +44,12 @@ func WithImmutables(immutables ...string) OptionModifier { } } +func WithExceptImmutables(exceptImmutables ...string) OptionModifier { + return func(p *Provider) { + p.exceptImmutables = append(p.exceptImmutables, exceptImmutables...) + } +} + func WithFlags(flags *pflag.FlagSet) OptionModifier { return func(p *Provider) { p.flags = flags diff --git a/configx/provider.go b/configx/provider.go index 6ae4c472..4e39e27b 100644 --- a/configx/provider.go +++ b/configx/provider.go @@ -39,7 +39,7 @@ type tuple struct { type Provider struct { l sync.RWMutex *koanf.Koanf - immutables []string + immutables, exceptImmutables []string schema []byte flags *pflag.FlagSet @@ -249,6 +249,18 @@ func (p *Provider) runOnChanges(e watcherx.Event, err error) { } } +func deleteOtherKeys(k *koanf.Koanf, keys []string) { +outer: + for _, key := range k.Keys() { + for _, ik := range keys { + if key == ik { + continue outer + } + } + k.Delete(key) + } +} + func (p *Provider) reload(e watcherx.Event) { p.l.Lock() @@ -264,10 +276,20 @@ func (p *Provider) reload(e watcherx.Event) { return // unlocks & runs changes in defer } - for _, key := range p.immutables { - if !reflect.DeepEqual(p.Koanf.Get(key), nk.Get(key)) { - err = NewImmutableError(key, fmt.Sprintf("%v", p.Koanf.Get(key)), fmt.Sprintf("%v", nk.Get(key))) - return // unlocks & runs changes in defer + oldImmutables, newImmutables := p.Koanf.Copy(), nk.Copy() + deleteOtherKeys(oldImmutables, p.immutables) + deleteOtherKeys(newImmutables, p.immutables) + + for _, key := range p.exceptImmutables { + oldImmutables.Delete(key) + newImmutables.Delete(key) + } + if !reflect.DeepEqual(oldImmutables.Raw(), newImmutables.Raw()) { + for _, key := range p.immutables { + if !reflect.DeepEqual(oldImmutables.Get(key), newImmutables.Get(key)) { + err = NewImmutableError(key, fmt.Sprintf("%v", p.Koanf.Get(key)), fmt.Sprintf("%v", nk.Get(key))) + return // unlocks & runs changes in defer + } } } diff --git a/configx/provider_watch_test.go b/configx/provider_watch_test.go index 0e4f4877..22dd1749 100644 --- a/configx/provider_watch_test.go +++ b/configx/provider_watch_test.go @@ -139,6 +139,31 @@ func TestReload(t *testing.T) { assertNoOpenFDs(t, dir, name) }) + t.Run("case=allows to update excepted immutable", func(t *testing.T) { + t.Parallel() + config := `{"foo": {"bar": "a", "baz": "b"}}` + + dir := t.TempDir() + name := "config.json" + watcherx.KubernetesAtomicWrite(t, dir, name, config) + + c := make(chan struct{}) + p, _ := setup(t, dir, name, c, + WithImmutables("foo"), + WithExceptImmutables("foo.baz"), + SkipValidation()) + + assert.Equal(t, "a", p.String("foo.bar")) + assert.Equal(t, "b", p.String("foo.baz")) + + config = `{"foo": {"bar": "a", "baz": "x"}}` + watcherx.KubernetesAtomicWrite(t, dir, name, config) + <-c + time.Sleep(time.Millisecond) + + assert.Equal(t, "x", p.String("foo.baz")) + }) + t.Run("case=runs without validation errors", func(t *testing.T) { t.Parallel() dir, name := tmpConfigFile(t, "some string", "bar") diff --git a/corsx/check_origin.go b/corsx/check_origin.go new file mode 100644 index 00000000..f5bf037f --- /dev/null +++ b/corsx/check_origin.go @@ -0,0 +1,54 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package corsx + +import "strings" + +// CheckOrigin is a function that can be used well with cors.Options.AllowOriginRequestFunc. +// It checks whether the origin is allowed following the same behavior as github.com/rs/cors. +// +// Recommended usage for hot-reloadable origins: +// +// func (p *Config) cors(ctx context.Context, prefix string) (cors.Options, bool) { +// opts, enabled := p.GetProvider(ctx).CORS(prefix, cors.Options{ +// AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE"}, +// AllowedHeaders: []string{"Authorization", "Content-Type", "Cookie"}, +// ExposedHeaders: []string{"Content-Type", "Set-Cookie"}, +// AllowCredentials: true, +// }) +// opts.AllowOriginRequestFunc = func(r *http.Request, origin string) bool { +// // load the origins from the config on every request to allow hot-reloading +// allowedOrigins := p.GetProvider(r.Context()).Strings(prefix + ".cors.allowed_origins") +// return corsx.CheckOrigin(allowedOrigins, origin) +// } +// return opts, enabled +// } +func CheckOrigin(allowedOrigins []string, origin string) bool { + if len(allowedOrigins) == 0 { + return true + } + for _, o := range allowedOrigins { + if o == "*" { + // allow all origins + return true + } + // Note: for origins and methods matching, the spec requires a case-sensitive matching. + // As it may be error-prone, we chose to ignore the spec here. + // https://github.com/rs/cors/blob/066574eebbd0f5f1b6cd1154a160cc292ac1835e/cors.go#L132-L133 + o = strings.ToLower(o) + prefix, suffix, found := strings.Cut(o, "*") + if !found { + // not a pattern, check for equality + if o == origin { + return true + } + continue + } + // inspired by https://github.com/rs/cors/blob/066574eebbd0f5f1b6cd1154a160cc292ac1835e/utils.go#L15 + if len(origin) >= len(prefix)+len(suffix) && strings.HasPrefix(origin, prefix) && strings.HasSuffix(origin, suffix) { + return true + } + } + return false +} diff --git a/corsx/check_origin_test.go b/corsx/check_origin_test.go new file mode 100644 index 00000000..f5d18774 --- /dev/null +++ b/corsx/check_origin_test.go @@ -0,0 +1,111 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package corsx + +import ( + "net/http" + "testing" + + "github.com/rs/cors" + "github.com/stretchr/testify/assert" +) + +func TestCheckOrigin(t *testing.T) { + for _, tc := range []struct { + name string + allowedOrigins []string + expect, expectOther bool + }{ + { + name: "empty", + allowedOrigins: []string{}, + expect: true, + expectOther: true, + }, + { + name: "wildcard", + allowedOrigins: []string{"https://example.com", "*"}, + expect: true, + expectOther: true, + }, + { + name: "exact", + allowedOrigins: []string{"https://www.ory.sh"}, + expect: true, + }, + { + name: "wildcard in the beginning", + allowedOrigins: []string{"*.ory.sh"}, + expect: true, + }, + { + name: "wildcard in the middle", + allowedOrigins: []string{"https://*.ory.sh"}, + expect: true, + }, + { + name: "wildcard in the end", + allowedOrigins: []string{"https://www.ory.*"}, + expect: true, + }, + { + name: "second wildcard is ignored", + allowedOrigins: []string{"https://*.ory.*"}, + expect: false, + }, + { + name: "multiple exact", + allowedOrigins: []string{"https://example.com", "https://www.ory.sh"}, + expect: true, + }, + { + name: "multiple wildcard", + allowedOrigins: []string{"https://*.example.com", "https://*.ory.sh"}, + expect: true, + }, + { + name: "wildcard and exact origins 1", + allowedOrigins: []string{"https://*.example.com", "https://www.ory.sh"}, + expect: true, + }, + { + name: "wildcard and exact origins 2", + allowedOrigins: []string{"https://example.com", "https://*.ory.sh"}, + expect: true, + }, + { + name: "multiple unrelated exact", + allowedOrigins: []string{"https://example.com", "https://example.org"}, + expect: false, + }, + { + name: "multiple unrelated with wildcard", + allowedOrigins: []string{"https://*.example.com", "https://*.example.org"}, + expect: false, + }, + { + name: "uppercase exact", + allowedOrigins: []string{"https://www.ORY.sh"}, + expect: true, + }, + { + name: "uppercase wildcard", + allowedOrigins: []string{"https://*.ORY.sh"}, + expect: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expect, CheckOrigin(tc.allowedOrigins, "https://www.ory.sh")) + + assert.Equal(t, tc.expectOther, CheckOrigin(tc.allowedOrigins, "https://google.com")) + + // check for consistency with rs/cors + assert.Equal(t, tc.expect, cors.New(cors.Options{AllowedOrigins: tc.allowedOrigins}). + OriginAllowed(&http.Request{Header: http.Header{"Origin": []string{"https://www.ory.sh"}}})) + + assert.Equal(t, tc.expectOther, cors.New(cors.Options{AllowedOrigins: tc.allowedOrigins}). + OriginAllowed(&http.Request{Header: http.Header{"Origin": []string{"https://google.com"}}})) + }) + } +} diff --git a/corsx/middleware.go b/corsx/middleware.go index fdbe59d1..a6ab0b82 100644 --- a/corsx/middleware.go +++ b/corsx/middleware.go @@ -19,6 +19,8 @@ import ( // panic("implement me") // }) // // ... +// +// Deprecated: because this is not really practical to use, you should use CheckOrigin as the cors.Options.AllowOriginRequestFunc instead. func ContextualizedMiddleware(provider func(context.Context) (opts cors.Options, enabled bool)) negroni.HandlerFunc { return func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { options, enabled := provider(r.Context()) From 7e34b1633fb9cd7420aa78af966541050b48c3fb Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:36:50 +0200 Subject: [PATCH 19/56] feat: add jwtmiddleware for oathkeeper JWTs (#715) --- go.mod | 3 + go.sum | 29 ++++++ jwtmiddleware/middleware.go | 123 ++++++++++++++++++++++++ jwtmiddleware/middleware_test.go | 154 +++++++++++++++++++++++++++++++ jwtmiddleware/stub/jwks.json | 10 ++ 5 files changed, 319 insertions(+) create mode 100644 jwtmiddleware/middleware.go create mode 100644 jwtmiddleware/middleware_test.go create mode 100644 jwtmiddleware/stub/jwks.json diff --git a/go.mod b/go.mod index 2727f516..6048ef26 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ory/x go 1.20 require ( + github.com/auth0/go-jwt-middleware v1.0.1 github.com/avast/retry-go/v4 v4.3.0 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 @@ -12,6 +13,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/evanphx/json-patch/v5 v5.6.0 github.com/fatih/structs v1.1.0 + github.com/form3tech-oss/jwt-go v3.2.2+incompatible github.com/fsnotify/fsnotify v1.6.0 github.com/ghodss/yaml v1.0.0 github.com/go-bindata/go-bindata v3.1.2+incompatible @@ -58,6 +60,7 @@ require ( github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.3.0 github.com/prometheus/common v0.37.0 + github.com/rakutentech/jwk-go v1.1.3 github.com/rs/cors v1.8.2 github.com/seatgeek/logrus-gelf-formatter v0.0.0-20210414080842-5b05eb8ff761 github.com/sirupsen/logrus v1.9.0 diff --git a/go.sum b/go.sum index ab51d274..2afd4a2f 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/auth0/go-jwt-middleware v1.0.1 h1:/fsQ4vRr4zod1wKReUH+0A3ySRjGiT9G34kypO/EKwI= +github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM= github.com/avast/retry-go/v4 v4.3.0 h1:cqI48aXx0BExKoM7XPklDpoHAg7/srPPLAfWG5z62jo= github.com/avast/retry-go/v4 v4.3.0/go.mod h1:bqOlT4nxk4phk9buiQFaghzjpqdchOSwPgjdfdQBtdg= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -182,6 +184,8 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -413,8 +417,12 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= @@ -459,6 +467,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -544,6 +553,7 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= @@ -674,11 +684,18 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nyaruka/phonenumbers v1.0.73/go.mod h1:3aiS+PS3DuYwkbK3xdcmRwMiPNECZ0oENH8qUT1lY7Q= github.com/nyaruka/phonenumbers v1.1.1 h1:fyoZmpLN2VCmAnc51XcrNOUVP2wT1ZzQl348ggIaXII= github.com/nyaruka/phonenumbers v1.1.1/go.mod h1:cGaEsOrLjIL0iKGqJR5Rfywy86dSkbApEpXuM9KySNA= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= @@ -760,6 +777,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rakutentech/jwk-go v1.1.3 h1:PiLwepKyUaW+QFG3ki78DIO2+b4IVK3nMhlxM70zrQ4= +github.com/rakutentech/jwk-go v1.1.3/go.mod h1:LtzSv4/+Iti1nnNeVQiP6l5cI74GBStbhyXCYvgPZFk= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -807,6 +826,9 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0= +github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d h1:yKm7XZV6j9Ev6lojP2XaIshpT4ymkqhMeSghO5Ps00E= @@ -970,6 +992,7 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= @@ -1032,6 +1055,7 @@ golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1110,6 +1134,7 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1132,6 +1157,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1399,6 +1425,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/mold.v2 v2.2.0/go.mod h1:XMyyRsGtakkDPbxXbrA5VODo6bUXyvoDjLd5l3T0XoA= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= @@ -1408,6 +1435,8 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/jwtmiddleware/middleware.go b/jwtmiddleware/middleware.go new file mode 100644 index 00000000..4ada135b --- /dev/null +++ b/jwtmiddleware/middleware.go @@ -0,0 +1,123 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package jwtmiddleware + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/form3tech-oss/jwt-go" + "github.com/pkg/errors" + + "github.com/ory/herodot" + + jwtmiddleware "github.com/auth0/go-jwt-middleware" + "github.com/urfave/negroni" + + "github.com/ory/x/jwksx" +) + +const SessionContextKey string = "github.com/ory/x/jwtmiddleware.session" + +type Middleware struct { + o *middlewareOptions + wku string + jm *jwtmiddleware.JWTMiddleware +} + +type middlewareOptions struct { + Debug bool + ExcludePaths []string + SigningMethod jwt.SigningMethod +} + +type MiddlewareOption func(*middlewareOptions) + +func SessionFromContext(ctx context.Context) (json.RawMessage, error) { + raw := ctx.Value(SessionContextKey) + if raw == nil { + return nil, errors.WithStack(herodot.ErrUnauthorized.WithReasonf("Could not find credentials in the request.")) + } + + token, ok := raw.(*jwt.Token) + if !ok { + return nil, errors.WithStack(herodot.ErrInternalServerError.WithDebugf(`Expected context key "%s" to transport value of type *jwt.MapClaims but got type: %T`, SessionContextKey, raw)) + } + + session, err := json.Marshal(token.Claims) + if err != nil { + return nil, errors.WithStack(herodot.ErrInternalServerError.WithDebugf("Unable to encode session data: %s", err)) + } + + return session, nil +} + +func MiddlewareDebugEnabled() MiddlewareOption { + return func(o *middlewareOptions) { + o.Debug = true + } +} + +func MiddlewareExcludePaths(paths ...string) MiddlewareOption { + return func(o *middlewareOptions) { + o.ExcludePaths = append(o.ExcludePaths, paths...) + } +} + +func MiddlewareAllowSigningMethod(method jwt.SigningMethod) MiddlewareOption { + return func(o *middlewareOptions) { + o.SigningMethod = method + } +} + +func NewMiddleware( + wellKnownURL string, + opts ...MiddlewareOption, +) *Middleware { + c := &middlewareOptions{ + SigningMethod: jwt.SigningMethodES256, + } + + for _, o := range opts { + o(c) + } + jc := jwksx.NewFetcher(wellKnownURL) + return &Middleware{ + o: c, + wku: wellKnownURL, + jm: jwtmiddleware.New(jwtmiddleware.Options{ + ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) { + if raw, ok := token.Header["kid"]; !ok { + return nil, errors.New(`jwt from authorization HTTP header is missing value for "kid" in token header`) + } else if kid, ok := raw.(string); !ok { + return nil, fmt.Errorf(`jwt from authorization HTTP header is expecting string value for "kid" in tokenWithoutKid header but got: %T`, raw) + } else if k, err := jc.GetKey(kid); err != nil { + return nil, err + } else { + return k.Key, nil + } + }, + SigningMethod: c.SigningMethod, + UserProperty: SessionContextKey, + CredentialsOptional: false, + Debug: c.Debug, + }), + } +} + +func (h *Middleware) NegroniHandler() negroni.Handler { + return negroni.HandlerFunc(func(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + for _, excluded := range h.o.ExcludePaths { + if strings.HasPrefix(r.URL.Path, excluded) { + next(w, r) + return + } + } + + h.jm.HandlerWithNext(w, r, next) + }) +} diff --git a/jwtmiddleware/middleware_test.go b/jwtmiddleware/middleware_test.go new file mode 100644 index 00000000..2639b59d --- /dev/null +++ b/jwtmiddleware/middleware_test.go @@ -0,0 +1,154 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package jwtmiddleware_test + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/form3tech-oss/jwt-go" + "github.com/rakutentech/jwk-go/jwk" + "github.com/stretchr/testify/assert" + + "github.com/ory/x/jwtmiddleware" + + _ "embed" + + "github.com/tidwall/sjson" + + "github.com/julienschmidt/httprouter" + "github.com/stretchr/testify/require" + "github.com/urfave/negroni" +) + +func mustString(s string, err error) string { + if err != nil { + panic(err) + } + return s +} + +var key *jwk.KeySpec + +//go:embed stub/jwks.json +var rawKey []byte + +func init() { + key = &jwk.KeySpec{} + if err := json.Unmarshal(rawKey, key); err != nil { + panic(err) + } +} + +func createToken(t *testing.T, claims jwt.MapClaims) string { + c := jwt.NewWithClaims(jwt.SigningMethodES256, claims) + c.Header["kid"] = key.KeyID + s, err := c.SignedString(key.Key) + require.NoError(t, err) + return s +} + +func newKeyServer(t *testing.T) string { + public, err := key.PublicOnly() + require.NoError(t, err) + keys, err := json.Marshal(map[string]interface{}{ + "keys": []interface{}{ + public, + }, + }) + require.NoError(t, err) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(keys) + })) + t.Cleanup(ts.Close) + return ts.URL +} + +func TestSessionFromRequest(t *testing.T) { + ks := newKeyServer(t) + + router := httprouter.New() + router.GET("/anonymous", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Write([]byte("ok")) + }) + router.GET("/me", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + s, err := jwtmiddleware.SessionFromContext(r.Context()) + require.NoError(t, err) + + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(s)) + }) + n := negroni.New() + n.Use(jwtmiddleware.NewMiddleware(ks, jwtmiddleware.MiddlewareExcludePaths("/anonymous")).NegroniHandler()) + n.UseHandler(router) + + ts := httptest.NewServer(n) + defer ts.Close() + + for k, tc := range []struct { + token string + expectedStatusCode int + expectedResponse string + }{ + // token without token + { + token: "", + expectedStatusCode: 401, + expectedResponse: "Authorization header format must be Bearer {token}", + }, + // token without kid + { + token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwiZXhwIjo5OTk5OTk5OTk5LCJzZXNzaW9uIjp7ImlkZW50aXR5Ijp7ImlkIjoiMTIzNDU2Nzg5MCJ9fX0.j0SgjC21nhkNP2QX0uE-I4wDYYRYlZq9wqGeDhrbplkKGW4BOjW5Sk0XFFbqrx68hQYz23QvYOYW5avUBzTjPxHwVqB1HPv6M5P2wHvRn7ZvAyhz83fmJMnBRNBOz1MfjxnEgkwfcVbNqsW2y37kRdZfveBlAzSfuPJV8Rkb4wlBbEGUwoCk78j8zcD_dcYFfXbt7uXz_tscScoIOg959Rmwr2E1XqRNy2qWLKSImwo8athdEEE-byLYytg6mgM02bmEQk2dyd5W2MmqG_4UaiBru6Bf9-drqExHDGUyndnAKi_uvF_131_LkPxy6H5Hu_YfZgSE5hXUbRsBzU-gbY5aV5FSn855PnRDyS_lFnBEn-0vcCIMmxbdfhqyKtFPmFHdSO1YsGruhqYaOLOlTVzThP-1XJSpgMKXHXW35c52zB9AaTV-0ETICvZ_OjZM_uzdWeb6PQmFsztcwdO-9C70yR3_HdcjljvnQ4XHs9ho_3_V57fcbW3uQCTq0TRbwD0AXpkVOvKJqaP1yEXYLKSNpGL2MMkuY-i3k6wTZMTV1280TqbJcSpY5n6WoWJnjoZ08BwBQDfX8AUsKk-D71wJbONqmLo5YnmrS-1gHR3bKCfuUzDdvensLXYJwSHg3ae_qE5VxscRhT_p2odeE8JgQBhd0d6765YBAP93F1c", + expectedStatusCode: 401, + expectedResponse: "jwt from authorization HTTP header is missing value for \"kid\" in token header", + }, + // token with int kid + { + token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6MTIzfQ.eyJzdWIiOiIxMjM0NTY3ODkwIiwiZXhwIjo5OTk5OTk5OTk5LCJzZXNzaW9uIjp7ImlkZW50aXR5Ijp7ImlkIjoiMTIzNDU2Nzg5MCJ9fX0.pG51ns8s_HeRC_KwtO7SNtIinqgVlSketJs7EjrHbW1xHvLRwCl4qhtIRuLqlED6eTEnqS2r2f6OFAiOJIZl9I6mQttSraHNcUOvK6t0bYg9w_K0HcaVu_894uJLZBTMx0B8mbqr7rZoRN_frriGkkjXbMP75-g1crA-t7_0VQeGwRPx0bcSF0T5yFRQyRlRwUTb6NbpLp6mc6NxMRP5OZPqnMTXAtP9YOfGLFdmhZ5CK1GUTdCRicwUyUOre8MNm4uIPZTTBZav06ncvjK80ATX7hkJqQfvvSlTee0LsLNHpuKPMCb_jmDaEugMXzvKPZ40L-r93KJ0TlK_dqu75imiK5aVuPaz8mk3cno4_0PW3ia0z5e00dWla1E8X1bOiW-4XvNdD1GGYGG0oBje67FnNFYQU2ApECbFN-3yGraneZFEcWWsf3CAEukcrmjjJLXYX0koUBtqvClOXHpKvwu-WhZ4eFYPoJoEysS4WeX7onxls2YdHsMBG9Ku-F26qzIHi1pDNsGb3eDbsGAMjaqEV81YfzwgBIF1nhfzuS0IU3LMoiwbwyQA6-hsAcV1dHTIoIW4VT1iEk90fsLzEMprh__SxYFIlOXchDWPD08sHLQk2kVLUR_BosdrygmTwkHVsq_lvIH77FsDkhwdKpD_sgdIdW_ttnYtCdMGlJc", + expectedStatusCode: 401, + expectedResponse: "jwt from authorization HTTP header is expecting string value for \"kid\" in tokenWithoutKid header but got: float64", + }, + // token with unknown kid + { + token: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Im5vdC1hLXZhbGlkLWtpZCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwiZXhwIjo5OTk5OTk5OTk5LCJzZXNzaW9uIjp7ImlkZW50aXR5Ijp7ImlkIjoiMTIzNDU2Nzg5MCJ9fX0.rX173fvU_Ed2p-iYF8PcRr4tS4e-BZR8RFV_CVtgEJxk2vMZHOlygJgvTZVK1cIP63EpHVqK_Sr5b1ctapLxpWMoxXBfdnyegZ5gLrDZ5vnbTJoWxpPo71D2RK2dC9qLwjBQr0MlYaLFUZrPcPOhsoYMlPTzLXamR0EGTY8lzPJhi3FubbnIWmq91v1ie-kF5d2Mxw_VnvF7ZJB5JwIH2KxkyVmGtImydmmkiXfuiNx1jejM68XW3mtfOFcuJYxc01jYR3l1Jh4E09hXNjYxqrR6oUjbmQZum60AInR_UyXw2myjkeAxj-m89ndm_z2MjrT0Za0cBuz0hY45FX6lOuANCCN6KOK3WmgdR6MCLxDWkNauicpMvsj14vF7V6W9kMpROE3YGxYySdG0ob8dtOurbYbFewFGi_ivmq7boMgwE1u6KpIKpW_DOjxCPcyP9UpxyAtFOGzV9cDUY_VA6rRWYktfBzE2HQpMPxX41FVhUT8Up0FGoUe1xnPkHLza17ZsGDVbfOMC-ji_kPRNi6rCZSn_nidr_7NbwhhaYkuPdWYtPLhr0XTsuwC2U0yGduwzP-ew8GiHQUvNBdio_WxhSHZm5WerFWzMB2_3QiMkh9O77axz1BmDGyXxs1OzUlvUKtPBlAz5b8oH_wdbGHiDfpL4c4qL_QAZfFpma4I", + expectedStatusCode: 401, + expectedResponse: "unable to find JSON Web Key with ID: not-a-valid-kid", + }, + // token with valid kid + { + token: createToken(t, jwt.MapClaims{ + "identity": map[string]interface{}{"email": "foo@bar.com"}, + }), + expectedStatusCode: 200, + expectedResponse: mustString(sjson.SetRaw("{}", "identity", "{\"email\":\"foo@bar.com\"}")), + }, + } { + t.Run(fmt.Sprintf("case=%d", k), func(t *testing.T) { + req, err := http.NewRequest("GET", ts.URL+"/me", nil) + require.NoError(t, err) + req.Header.Set("Authorization", "bearer "+tc.token) + require.NoError(t, err) + + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + assert.Equal(t, tc.expectedStatusCode, res.StatusCode, string(body)) + assert.Equal(t, tc.expectedResponse, strings.TrimSpace(string(body))) + }) + } + + res, err := http.Get(ts.URL + "/anonymous") + require.NoError(t, err) + assert.Equal(t, 200, res.StatusCode) +} diff --git a/jwtmiddleware/stub/jwks.json b/jwtmiddleware/stub/jwks.json new file mode 100644 index 00000000..57d130c4 --- /dev/null +++ b/jwtmiddleware/stub/jwks.json @@ -0,0 +1,10 @@ +{ + "use": "sig", + "kty": "EC", + "kid": "b71ff5bd-a016-4ac0-9f3f-a172552578ea", + "crv": "P-256", + "alg": "ES256", + "x": "7fVj_SeCx3TnkHANRWrpEho9BcYkU953LHUvKsSF5Wo", + "y": "2A9D_AAFPiJQLSJQ_h600Fy9jUrg9Q88gNPPZwHDb7o", + "d": "sRl-e-tGEVsNBF8FgEado9NAEipxhAFXGMryWDgbUMo" +} From b50b315a79972cf40bdd620e30aa00b140c3f1ab Mon Sep 17 00:00:00 2001 From: Patrik Date: Fri, 11 Aug 2023 11:48:54 +0200 Subject: [PATCH 20/56] fix(pagination): quote function broken for mysql (#716) --- pagination/keysetpagination/paginator.go | 5 +++-- pagination/keysetpagination/paginator_test.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pagination/keysetpagination/paginator.go b/pagination/keysetpagination/paginator.go index c6ec4bd1..a69a8f64 100644 --- a/pagination/keysetpagination/paginator.go +++ b/pagination/keysetpagination/paginator.go @@ -132,10 +132,11 @@ func Paginate[I any, PI interface { id := model.IDField() tableName := model.Alias() return func(q *pop.Query) *pop.Query { - eid := q.Connection.Dialect.Quote(tableName + "." + id) + quote := q.Connection.Dialect.Quote + eid := quote(tableName) + "." + quote(id) quoteAndContextualize := func(name string) string { - return q.Connection.Dialect.Quote(tableName + "." + name) + return quote(tableName) + "." + quote(name) } p.multipleOrderFieldsQuery(q, id, model.Columns().Cols, quoteAndContextualize) diff --git a/pagination/keysetpagination/paginator_test.go b/pagination/keysetpagination/paginator_test.go index d4c5c457..bbc5d1af 100644 --- a/pagination/keysetpagination/paginator_test.go +++ b/pagination/keysetpagination/paginator_test.go @@ -49,7 +49,7 @@ func TestPaginator(t *testing.T) { q = q.Scope(Paginate[testItem](paginator)) sql, args := q.ToSQL(&pop.Model{Value: new(testItem)}) - assert.Equal(t, "SELECT test_items.created_at, test_items.pk FROM test_items AS test_items WHERE `test_items.pk` > ? ORDER BY `test_items.pk` ASC LIMIT 11", sql) + assert.Equal(t, "SELECT test_items.created_at, test_items.pk FROM test_items AS test_items WHERE `test_items`.`pk` > ? ORDER BY `test_items`.`pk` ASC LIMIT 11", sql) assert.Equal(t, []interface{}{"token"}, args) }) From a20fd1ec89bbbd3b9fd3ee84102b155c1f245f57 Mon Sep 17 00:00:00 2001 From: Patrik Date: Fri, 11 Aug 2023 12:09:53 +0200 Subject: [PATCH 21/56] docs(pagination): improve parameter descriptions (#712) --- pagination/migrationpagination/header.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pagination/migrationpagination/header.go b/pagination/migrationpagination/header.go index d78b6d29..0a24143c 100644 --- a/pagination/migrationpagination/header.go +++ b/pagination/migrationpagination/header.go @@ -22,11 +22,11 @@ type RequestParameters struct { // reference. The next page can be any number and some numbers might return an empty list. // // For example, page 2 might not follow after page 1. And even if page 3 and 5 exist, but page 4 might not exist. + // The first page can be retrieved by omitting this parameter. Following page pointers will be returned in the + // `Link` header. // // required: false // in: query - // default: 1 - // min: 1 Page int `json:"page"` } @@ -58,5 +58,8 @@ type ResponseHeaderAnnotation struct { // The X-Total-Count HTTP Header // // The `X-Total-Count` header contains the total number of items in the collection. + // + // DEPRECATED: This header will be removed eventually. Please use the `Link` header + // instead to check whether you are on the last page. TotalCount int `json:"x-total-count"` } From 3ec4565cd890fe54f73d84d3549e1be2e358883f Mon Sep 17 00:00:00 2001 From: Henning Perl Date: Wed, 16 Aug 2023 14:18:07 +0200 Subject: [PATCH 22/56] feat: match exception URLs via globbing (#717) --- go.mod | 2 ++ go.sum | 2 ++ httpx/private_ip_validator.go | 7 ++++++- httpx/resilient_client.go | 6 +++--- httpx/resilient_client_test.go | 17 ++++++++++------- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 6048ef26..a61e7c4e 100644 --- a/go.mod +++ b/go.mod @@ -95,6 +95,8 @@ require ( google.golang.org/protobuf v1.30.0 ) +require github.com/gobwas/glob v0.2.3 // indirect + require ( git.sr.ht/~sbinet/gg v0.3.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect diff --git a/go.sum b/go.sum index 2afd4a2f..82d8e45b 100644 --- a/go.sum +++ b/go.sum @@ -321,6 +321,8 @@ github.com/gobuffalo/tags/v3 v3.1.4 h1:X/ydLLPhgXV4h04Hp2xlbI2oc5MDaa7eub6zw8oHj github.com/gobuffalo/tags/v3 v3.1.4/go.mod h1:ArRNo3ErlHO8BtdA0REaZxijuWnWzF6PUXngmMXd2I0= github.com/gobuffalo/validate/v3 v3.3.3 h1:o7wkIGSvZBYBd6ChQoLxkz2y1pfmhbI4jNJYh6PuNJ4= github.com/gobuffalo/validate/v3 v3.3.3/go.mod h1:YC7FsbJ/9hW/VjQdmXPvFqvRis4vrRYFxr69WiNZw6g= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.9.6 h1:KhAu1zf9JXnm3vbG49aDE0E5uEBUsM4uwD31/58ZWyI= github.com/goccy/go-yaml v1.9.6/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= diff --git a/httpx/private_ip_validator.go b/httpx/private_ip_validator.go index df0f825b..09e20732 100644 --- a/httpx/private_ip_validator.go +++ b/httpx/private_ip_validator.go @@ -11,6 +11,7 @@ import ( "syscall" "time" + "github.com/gobwas/glob" "github.com/pkg/errors" ) @@ -94,7 +95,11 @@ func (n NoInternalIPRoundTripper) RoundTrip(request *http.Request) (*http.Respon incoming.RawQuery = "" incoming.RawFragment = "" for _, exception := range n.internalIPExceptions { - if incoming.String() == exception { + compiled, err := glob.Compile(exception, '.', '/') + if err != nil { + return nil, err + } + if compiled.Match(incoming.String()) { return rt.RoundTrip(request) } } diff --git a/httpx/resilient_client.go b/httpx/resilient_client.go index 9e632235..4ab77276 100644 --- a/httpx/resilient_client.go +++ b/httpx/resilient_client.go @@ -98,11 +98,11 @@ func ResilientClientDisallowInternalIPs() ResilientOptions { } } -// ResilientClientAllowInternalIPRequestsTo allows requests to the exact matching URLs even +// ResilientClientAllowInternalIPRequestsTo allows requests to the glob-matching URLs even // if they are internal IPs. -func ResilientClientAllowInternalIPRequestsTo(urls ...string) ResilientOptions { +func ResilientClientAllowInternalIPRequestsTo(urlGlobs ...string) ResilientOptions { return func(o *resilientOptions) { - o.internalIPExceptions = urls + o.internalIPExceptions = urlGlobs } } diff --git a/httpx/resilient_client_test.go b/httpx/resilient_client_test.go index e7891fd7..8dc4fc8d 100644 --- a/httpx/resilient_client_test.go +++ b/httpx/resilient_client_test.go @@ -28,20 +28,23 @@ func TestNoPrivateIPs(t *testing.T) { _, port, err := net.SplitHostPort(target.Host) require.NoError(t, err) - allowed := "http://localhost:" + port + "/foobar" + allowedURL := "http://localhost:" + port + "/foobar" + allowedGlob := "http://localhost:" + port + "/glob/*" c := NewResilientClient( ResilientClientWithMaxRetry(1), ResilientClientDisallowInternalIPs(), - ResilientClientAllowInternalIPRequestsTo(allowed), + ResilientClientAllowInternalIPRequestsTo(allowedURL, allowedGlob), ) for destination, passes := range map[string]bool{ - "http://127.0.0.1:" + port: false, - "http://localhost:" + port: false, - "http://192.168.178.5:" + port: false, - allowed: true, - "http://localhost:" + port + "/FOOBAR": false, + "http://127.0.0.1:" + port: false, + "http://localhost:" + port: false, + "http://192.168.178.5:" + port: false, + allowedURL: true, + "http://localhost:" + port + "/glob/bar": true, + "http://localhost:" + port + "/glob/bar/baz": false, + "http://localhost:" + port + "/FOOBAR": false, } { _, err := c.Get(destination) if !passes { From 460bc7411a9880cc87bea55371984ea80915dc35 Mon Sep 17 00:00:00 2001 From: Patrik Date: Fri, 18 Aug 2023 14:57:59 +0200 Subject: [PATCH 23/56] chore(jsonnetsecure): add tracing (#718) --- jsonnetsecure/jsonnet.go | 2 +- jsonnetsecure/jsonnet_processvm.go | 23 ++++++++++++++++++----- jsonnetsecure/jsonnet_test.go | 9 ++++++--- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/jsonnetsecure/jsonnet.go b/jsonnetsecure/jsonnet.go index a8fe627a..69efc9fa 100644 --- a/jsonnetsecure/jsonnet.go +++ b/jsonnetsecure/jsonnet.go @@ -111,7 +111,7 @@ func JsonnetTestBinary(t testing.TB) string { cmd.Stderr = &stderr if err := cmd.Run(); err != nil || stderr.Len() != 0 { - t.Fatalf("building the Go binary returned error: %v\n%s", err, string(stderr.String())) + t.Fatalf("building the Go binary returned error: %v\n%s", err, stderr.String()) } return outPath diff --git a/jsonnetsecure/jsonnet_processvm.go b/jsonnetsecure/jsonnet_processvm.go index 391948b0..f057156f 100644 --- a/jsonnetsecure/jsonnet_processvm.go +++ b/jsonnetsecure/jsonnet_processvm.go @@ -13,6 +13,12 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" + + "github.com/ory/x/otelx" + + "go.opentelemetry.io/otel/trace" + "github.com/cenkalti/backoff/v4" "github.com/pkg/errors" ) @@ -25,11 +31,18 @@ func NewProcessVM(opts *vmOptions) VM { } } -func (p *ProcessVM) EvaluateAnonymousSnippet(filename string, snippet string) (string, error) { +func (p *ProcessVM) EvaluateAnonymousSnippet(filename string, snippet string) (_ string, err error) { + tracer := trace.SpanFromContext(p.ctx).TracerProvider().Tracer("") + ctx, span := tracer.Start(p.ctx, "jsonnetsecure.ProcessVM.EvaluateAnonymousSnippet", trace.WithAttributes(attribute.String("filename", filename))) + defer otelx.End(span, &err) + // We retry the process creation, because it sometimes times out. const processVMTimeout = 1 * time.Second - return backoff.RetryWithData(func() (string, error) { - ctx, cancel := context.WithTimeout(p.ctx, processVMTimeout) + return backoff.RetryWithData(func() (_ string, err error) { + ctx, span := tracer.Start(ctx, "jsonnetsecure.ProcessVM.EvaluateAnonymousSnippet.run") + defer otelx.End(span, &err) + + ctx, cancel := context.WithTimeout(ctx, processVMTimeout) defer cancel() var ( @@ -49,7 +62,7 @@ func (p *ProcessVM) EvaluateAnonymousSnippet(filename string, snippet string) (s cmd.Stderr = &stderr cmd.Env = []string{"GOMAXPROCS=1"} - err := cmd.Run() + err = cmd.Run() if stderr.Len() > 0 { // If the process wrote to stderr, this means it started and we won't retry. return "", backoff.Permanent(fmt.Errorf("jsonnetsecure: unexpected output on stderr: %q", stderr.String())) @@ -59,7 +72,7 @@ func (p *ProcessVM) EvaluateAnonymousSnippet(filename string, snippet string) (s } return stdout.String(), nil - }, backoff.WithContext(backoff.NewExponentialBackOff(), p.ctx)) + }, backoff.WithContext(backoff.NewExponentialBackOff(), ctx)) } func (p *ProcessVM) ExtCode(key string, val string) { diff --git a/jsonnetsecure/jsonnet_test.go b/jsonnetsecure/jsonnet_test.go index 56d8ea68..1904d055 100644 --- a/jsonnetsecure/jsonnet_test.go +++ b/jsonnetsecure/jsonnet_test.go @@ -18,6 +18,8 @@ import ( ) func TestSecureVM(t *testing.T) { + testBinary := JsonnetTestBinary(t) + for _, optCase := range []struct { name string opts []Option @@ -25,7 +27,7 @@ func TestSecureVM(t *testing.T) { {"none", []Option{}}, {"process vm", []Option{ WithProcessIsolatedVM(context.Background()), - WithJsonnetBinary(JsonnetTestBinary(t)), + WithJsonnetBinary(testBinary), }}, } { t.Run("options="+optCase.name, func(t *testing.T) { @@ -114,7 +116,7 @@ func TestSecureVM(t *testing.T) { defer cancel() vm := MakeSecureVM( WithProcessIsolatedVM(ctx), - WithJsonnetBinary(JsonnetTestBinary(t)), + WithJsonnetBinary(testBinary), ) result, err := vm.EvaluateAnonymousSnippet("test", snippet) require.Error(t, err) @@ -161,12 +163,13 @@ func assertEqualVMOutput(t *testing.T, run func(factory func(t *testing.T) VM) s func TestCreateMultipleProcessVMs(t *testing.T) { ctx := context.Background() wg := new(errgroup.Group) + testBinary := JsonnetTestBinary(t) for i := 0; i < 100; i++ { wg.Go(func() error { vm := MakeSecureVM( WithProcessIsolatedVM(ctx), - WithJsonnetBinary(JsonnetTestBinary(t)), + WithJsonnetBinary(testBinary), ) _, err := vm.EvaluateAnonymousSnippet("test", "{a:1}") From b0b6aa77b18579a02dda5bfbb3ddf800eb7078f6 Mon Sep 17 00:00:00 2001 From: Sergey Plaunov Date: Tue, 22 Aug 2023 13:59:38 +0300 Subject: [PATCH 24/56] feat: add `deployment.environment` attribute to OTEL data (CORE-2294) (#710) --- configx/provider.go | 5 +++-- otelx/config.go | 7 ++++--- otelx/config.schema.json | 5 +++++ otelx/otlp.go | 1 + 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/configx/provider.go b/configx/provider.go index 4e39e27b..fd7c1b15 100644 --- a/configx/provider.go +++ b/configx/provider.go @@ -454,8 +454,9 @@ func (p *Provider) CORS(prefix string, defaults cors.Options) (cors.Options, boo func (p *Provider) TracingConfig(serviceName string) *otelx.Config { return &otelx.Config{ - ServiceName: p.StringF("tracing.service_name", serviceName), - Provider: p.String("tracing.provider"), + ServiceName: p.StringF("tracing.service_name", serviceName), + DeploymentEnvironment: p.StringF("tracing.deployment_environment", ""), + Provider: p.String("tracing.provider"), Providers: otelx.ProvidersConfig{ Jaeger: otelx.JaegerConfig{ Sampling: otelx.JaegerSampling{ diff --git a/otelx/config.go b/otelx/config.go index 32c2ec56..812fe2a3 100644 --- a/otelx/config.go +++ b/otelx/config.go @@ -46,9 +46,10 @@ type ProvidersConfig struct { } type Config struct { - ServiceName string `json:"service_name"` - Provider string `json:"provider"` - Providers ProvidersConfig `json:"providers"` + ServiceName string `json:"service_name"` + DeploymentEnvironment string `json:"deployment_environment"` + Provider string `json:"provider"` + Providers ProvidersConfig `json:"providers"` } //go:embed config.schema.json diff --git a/otelx/config.schema.json b/otelx/config.schema.json index 2593b89d..a53cd8da 100644 --- a/otelx/config.schema.json +++ b/otelx/config.schema.json @@ -16,6 +16,11 @@ "description": "Specifies the service name to use on the tracer.", "examples": ["Ory Hydra", "Ory Kratos", "Ory Keto", "Ory Oathkeeper"] }, + "deployment_environment": { + "type": "string", + "description": "Specifies the deployment environment to use on the tracer.", + "examples": ["development", "staging", "production"] + }, "providers": { "type": "object", "additionalProperties": false, diff --git a/otelx/otlp.go b/otelx/otlp.go index e05db7c7..c39e92fa 100644 --- a/otelx/otlp.go +++ b/otelx/otlp.go @@ -47,6 +47,7 @@ func SetupOTLP(t *Tracer, tracerName string, c *Config) (trace.Tracer, error) { sdktrace.WithResource(resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(c.ServiceName), + semconv.DeploymentEnvironmentKey.String(c.DeploymentEnvironment), )), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased( c.Providers.OTLP.Sampling.SamplingRatio, From 577e4fb62975a05ca9b973b2004b4b43d77573e9 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Fri, 25 Aug 2023 08:59:06 +0200 Subject: [PATCH 25/56] feat: add DirtyPatch method (#719) It's fast: ``` BenchmarkSet BenchmarkSet-8 6189 9197403 ns/op BenchmarkDirtyPatch BenchmarkDirtyPatch-8 401864 3057 ns/op ``` --- configx/provider.go | 27 ++++++++++++++++++++++ configx/provider_test.go | 48 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/configx/provider.go b/configx/provider.go index fd7c1b15..b5d5f13a 100644 --- a/configx/provider.go +++ b/configx/provider.go @@ -314,6 +314,33 @@ func (p *Provider) watchForFileChanges(ctx context.Context, c watcherx.EventChan } } +// DirtyPatch patches individual config keys without reloading the full config +// +// WARNING! This method is only useful to override existing keys in string or number +// format. DO NOT use this method to override arrays, maps, or other complex types. +// +// This method DOES NOT validate the config against the config JSON schema. If you +// need to validate the config, use the Set method instead. +// +// This method can not be used to remove keys from the config as that is not +// possible without reloading the full config. +func (p *Provider) DirtyPatch(key string, value any) error { + p.l.Lock() + defer p.l.Unlock() + + t := tuple{Key: key, Value: value} + kc := NewKoanfConfmap([]tuple{t}) + + p.forcedValues = append(p.forcedValues, t) + p.providers = append(p.providers, kc) + + if err := p.Koanf.Load(kc, nil, []koanf.Option{}...); err != nil { + return err + } + + return nil +} + func (p *Provider) Set(key string, value interface{}) error { p.l.Lock() defer p.l.Unlock() diff --git a/configx/provider_test.go b/configx/provider_test.go index eeaba107..caa0cc81 100644 --- a/configx/provider_test.go +++ b/configx/provider_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestProviderMethods(t *testing.T) { +func newProvider(t testing.TB) *Provider { // Fake some flags f := pflag.NewFlagSet("config", pflag.ContinueOnError) f.String("foo-bar-baz", "", "") @@ -32,10 +32,15 @@ func TestProviderMethods(t *testing.T) { RegisterFlags(f) ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + t.Cleanup(cancel) p, err := New(ctx, []byte(`{"type": "object", "properties": {"foo-bar-baz": {"type": "string"}, "b": {"type": "string"}}}`), WithFlags(f), WithContext(ctx)) require.NoError(t, err) + return p +} + +func TestProviderMethods(t *testing.T) { + p := newProvider(t) t.Run("check flags", func(t *testing.T) { assert.Equal(t, "fff", p.String("foo-bar-baz")) @@ -106,6 +111,21 @@ func TestProviderMethods(t *testing.T) { assert.NoError(t, p.Set("nested.value", "https://www.ory.sh/kratos")) assert.Equal(t, "https://www.ory.sh/kratos", p.Get("nested.value")) }) + + t.Run("use DirtyPatch operations", func(t *testing.T) { + assert.NoError(t, p.DirtyPatch("nested", nil)) + assert.NoError(t, p.DirtyPatch("nested.value", "https://www.ory.sh/kratos")) + assert.Equal(t, "https://www.ory.sh/kratos", p.Get("nested.value")) + + assert.NoError(t, p.DirtyPatch("duration.integer1", -1)) + assert.NoError(t, p.DirtyPatch("duration.integer2", "-1")) + assert.Equal(t, -1*time.Nanosecond, p.DurationF("duration.integer1", time.Second)) + assert.Equal(t, -1*time.Nanosecond, p.DurationF("duration.integer2", time.Second)) + + require.NoError(t, p.DirtyPatch("some.float", 123.123)) + assert.Equal(t, 123.123, p.Float64F("some.float", 321.321)) + assert.Equal(t, 321.321, p.Float64F("not.some.float", 321.321)) + }) } func TestAdvancedConfigs(t *testing.T) { @@ -212,3 +232,27 @@ func TestAdvancedConfigs(t *testing.T) { }) } } + +func BenchmarkSet(b *testing.B) { + // Benchmark set function + p := newProvider(b) + var err error + for i := 0; i < b.N; i++ { + err = p.Set("nested.value", "https://www.ory.sh/kratos") + if err != nil { + b.Fatalf("Unexpected error: %s", err) + } + } +} + +func BenchmarkDirtyPatch(b *testing.B) { + // Benchmark set function + p := newProvider(b) + var err error + for i := 0; i < b.N; i++ { + err = p.DirtyPatch("nested.value", "https://www.ory.sh/kratos") + if err != nil { + b.Fatalf("Unexpected error: %s", err) + } + } +} From 5ebc1cb1ddeddbf833a46c2d8a482a11c8feef33 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:12:59 +0300 Subject: [PATCH 26/56] feat: add jwks fetcher (#720) Adds an efficient fetcher for JSON Web Key Sets. The fetcher is able to cache results for a given TTL, if desired. --- fetcher/fetcher.go | 4 +- go.mod | 45 ++-- go.sum | 94 +++++--- ...ce_urls-case=succeeds_with_forced_kid.json | 7 + ...lve_single_source_url-case=with_cache.json | 7 + ...le_source_url-case=with_cache_and_TTL.json | 7 + ...ingle_source_url-case=with_forced_key.json | 7 + ..._single_source_url-case=without_cache.json | 7 + jwksx/fetcher.go | 6 + jwksx/fetcher_v2.go | 167 ++++++++++++++ jwksx/fetcher_v2_test.go | 208 ++++++++++++++++++ 11 files changed, 509 insertions(+), 50 deletions(-) create mode 100644 jwksx/.snapshots/TestFetcherNext-case=resolve_multiple_source_urls-case=succeeds_with_forced_kid.json create mode 100644 jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_cache.json create mode 100644 jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_cache_and_TTL.json create mode 100644 jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_forced_key.json create mode 100644 jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=without_cache.json create mode 100644 jwksx/fetcher_v2.go create mode 100644 jwksx/fetcher_v2_test.go diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 867a718e..b58f0f51 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -44,8 +44,10 @@ func newOpts() *opts { } } +type Modifier func(*opts) + // NewFetcher creates a new fetcher instance. -func NewFetcher(opts ...func(*opts)) *Fetcher { +func NewFetcher(opts ...Modifier) *Fetcher { o := newOpts() for _, f := range opts { f(o) diff --git a/go.mod b/go.mod index a61e7c4e..d31af759 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/evanphx/json-patch/v5 v5.6.0 github.com/fatih/structs v1.1.0 - github.com/form3tech-oss/jwt-go v3.2.2+incompatible + github.com/form3tech-oss/jwt-go v3.2.5+incompatible github.com/fsnotify/fsnotify v1.6.0 github.com/ghodss/yaml v1.0.0 github.com/go-bindata/go-bindata v3.1.2+incompatible @@ -24,6 +24,7 @@ require ( github.com/gobuffalo/fizz v1.14.4 github.com/gobuffalo/httptest v1.5.2 github.com/gobuffalo/pop/v6 v6.0.8 + github.com/gobwas/glob v0.2.3 github.com/goccy/go-yaml v1.9.6 github.com/gofrs/uuid v4.3.0+incompatible github.com/golang/mock v1.6.0 @@ -44,6 +45,8 @@ require ( github.com/knadh/koanf/providers/posflag v0.1.0 github.com/knadh/koanf/providers/rawbytes v0.1.0 github.com/knadh/koanf/v2 v2.0.1 + github.com/laher/mergefs v0.1.1 + github.com/lestrrat-go/jwx v1.2.26 github.com/lib/pq v1.10.7 github.com/luna-duclos/instrumentedsql v1.1.3 github.com/markbates/pkger v0.17.1 @@ -52,7 +55,7 @@ require ( github.com/ory/analytics-go/v5 v5.0.1 github.com/ory/dockertest/v3 v3.9.1 github.com/ory/go-acc v0.2.9-0.20230103102148-6b1c9a70dbbe - github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88 + github.com/ory/herodot v0.9.13 github.com/ory/jsonschema/v3 v3.0.7 github.com/pelletier/go-toml v1.9.5 github.com/pkg/errors v0.9.1 @@ -65,9 +68,9 @@ require ( github.com/seatgeek/logrus-gelf-formatter v0.0.0-20210414080842-5b05eb8ff761 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cast v1.5.0 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.3 github.com/tidwall/pretty v1.2.1 github.com/tidwall/sjson v1.2.5 @@ -86,17 +89,15 @@ require ( go.opentelemetry.io/otel/trace v1.11.1 go.opentelemetry.io/proto/otlp v0.18.0 go.uber.org/goleak v1.2.1 - golang.org/x/crypto v0.1.0 - golang.org/x/mod v0.10.0 - golang.org/x/net v0.8.0 + golang.org/x/crypto v0.9.0 + golang.org/x/mod v0.8.0 + golang.org/x/net v0.10.0 golang.org/x/sync v0.1.0 gonum.org/v1/plot v0.12.0 - google.golang.org/grpc v1.54.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/grpc v1.53.0 + google.golang.org/protobuf v1.28.1 ) -require github.com/gobwas/glob v0.2.3 // indirect - require ( git.sr.ht/~sbinet/gg v0.3.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect @@ -110,6 +111,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/cli v20.10.21+incompatible // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect @@ -135,11 +137,12 @@ require ( github.com/gobuffalo/plush/v4 v4.1.16 // indirect github.com/gobuffalo/tags/v3 v3.1.4 // indirect github.com/gobuffalo/validate/v3 v3.3.3 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/glog v1.0.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/google/pprof v0.0.0-20221010195024-131d412537ea // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.0 // indirect @@ -157,7 +160,11 @@ require ( github.com/joho/godotenv v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/laher/mergefs v0.1.1 + github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect + github.com/lestrrat-go/blackmagic v1.0.1 // indirect + github.com/lestrrat-go/httpcc v1.0.1 // indirect + github.com/lestrrat-go/iter v1.0.2 // indirect + github.com/lestrrat-go/option v1.0.1 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -183,11 +190,11 @@ require ( github.com/sergi/go-diff v1.2.0 // indirect github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e // indirect - github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.14.0 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/subosito/gotenv v1.4.2 // indirect + github.com/subosito/gotenv v1.4.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -197,12 +204,12 @@ require ( go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect go.opentelemetry.io/otel/metric v0.33.0 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 82d8e45b..6c964ec1 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,9 @@ github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -184,8 +187,9 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -323,6 +327,8 @@ github.com/gobuffalo/validate/v3 v3.3.3 h1:o7wkIGSvZBYBd6ChQoLxkz2y1pfmhbI4jNJYh github.com/gobuffalo/validate/v3 v3.3.3/go.mod h1:YC7FsbJ/9hW/VjQdmXPvFqvRis4vrRYFxr69WiNZw6g= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-yaml v1.9.6 h1:KhAu1zf9JXnm3vbG49aDE0E5uEBUsM4uwD31/58ZWyI= github.com/goccy/go-yaml v1.9.6/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -370,9 +376,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -476,6 +481,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1: github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= @@ -602,6 +608,19 @@ github.com/laher/mergefs v0.1.1 h1:nV2bTS57vrmbMxeR6uvJpI8LyGl3QHj4bLBZO3aUV58= github.com/laher/mergefs v0.1.1/go.mod h1:FSY1hYy94on4Tz60waRMGdO1awwS23BacqJlqf9lJ9Q= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A= +github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= +github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80= +github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= +github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= +github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= +github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= +github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= +github.com/lestrrat-go/jwx v1.2.26 h1:4iFo8FPRZGDYe1t19mQP0zTRqA7n8HnJ5lkIiDvJcB0= +github.com/lestrrat-go/jwx v1.2.26/go.mod h1:MaiCdGbn3/cckbOFSCluJlJMmp9dmZm5hDuIkx8ftpQ= +github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= +github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -625,6 +644,7 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI github.com/markbates/pkger v0.17.1 h1:/MKEtWqtc0mZvu9OinB9UzVN9iYCwLWuyUv4Bw+PCno= github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -716,8 +736,8 @@ github.com/ory/dockertest/v3 v3.9.1/go.mod h1:42Ir9hmvaAPm0Mgibk6mBPi7SFvTXxEcnz github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= github.com/ory/go-acc v0.2.9-0.20230103102148-6b1c9a70dbbe h1:rvu4obdvqR0fkSIJ8IfgzKOWwZ5kOT2UNfLq81Qk7rc= github.com/ory/go-acc v0.2.9-0.20230103102148-6b1c9a70dbbe/go.mod h1:z4n3u6as84LbV4YmgjHhnwtccQqzf4cZlSk9f1FhygI= -github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88 h1:J0CIFKdpUeqKbVMw7pQ1qLtUnflRM1JWAcOEq7Hp4yg= -github.com/ory/herodot v0.10.3-0.20230626083119-d7e5192f0d88/go.mod h1:MMNmY6MG1uB6fnXYFaHoqdV23DTWctlPsmRCeq/2+wc= +github.com/ory/herodot v0.9.13 h1:cN/Z4eOkErl/9W7hDIDLb79IO/bfsH+8yscBjRpB4IU= +github.com/ory/herodot v0.9.13/go.mod h1:IWDs9kSvFQqw/cQ8zi5ksyYvITiUU4dI7glUrhZcJYo= github.com/ory/jsonschema/v3 v3.0.7 h1:GQ9qfZDiJqs4l2d3p56dozCChvejQFZyLKGHYzDzOSo= github.com/ory/jsonschema/v3 v3.0.7/go.mod h1:g8c8YOtN4TrR2wYeMdT02GDmzJDI0fEW2nI26BECafY= github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= @@ -733,6 +753,7 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= @@ -841,8 +862,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.5.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= -github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.2-0.20200723214538-8d17101741c8/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -852,8 +873,8 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -878,11 +899,12 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= -github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= @@ -924,6 +946,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1003,11 +1026,12 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1053,8 +1077,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1097,14 +1121,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1199,6 +1225,7 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1215,12 +1242,15 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1231,8 +1261,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1301,9 +1331,10 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1370,6 +1401,7 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1378,9 +1410,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1402,9 +1435,10 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99 h1:qA8rMbz1wQ4DOFfM2ouD29DG9aHWBm6ZOy9BGxiUMmY= +google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1418,8 +1452,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/jwksx/.snapshots/TestFetcherNext-case=resolve_multiple_source_urls-case=succeeds_with_forced_kid.json b/jwksx/.snapshots/TestFetcherNext-case=resolve_multiple_source_urls-case=succeeds_with_forced_kid.json new file mode 100644 index 00000000..ecb9a86a --- /dev/null +++ b/jwksx/.snapshots/TestFetcherNext-case=resolve_multiple_source_urls-case=succeeds_with_forced_kid.json @@ -0,0 +1,7 @@ +{ + "alg": "HS256", + "k": "Y2hhbmdlbWVjaGFuZ2VtZWNoYW5nZW1lY2hhbmdlbWU", + "kid": "8d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8", + "kty": "oct", + "use": "sig" +} diff --git a/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_cache.json b/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_cache.json new file mode 100644 index 00000000..f81e76cc --- /dev/null +++ b/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_cache.json @@ -0,0 +1,7 @@ +{ + "alg": "HS256", + "k": "Y2hhbmdlbWVjaGFuZ2VtZWNoYW5nZW1lY2hhbmdlbWU", + "kid": "7d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8", + "kty": "oct", + "use": "sig" +} diff --git a/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_cache_and_TTL.json b/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_cache_and_TTL.json new file mode 100644 index 00000000..f81e76cc --- /dev/null +++ b/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_cache_and_TTL.json @@ -0,0 +1,7 @@ +{ + "alg": "HS256", + "k": "Y2hhbmdlbWVjaGFuZ2VtZWNoYW5nZW1lY2hhbmdlbWU", + "kid": "7d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8", + "kty": "oct", + "use": "sig" +} diff --git a/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_forced_key.json b/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_forced_key.json new file mode 100644 index 00000000..f81e76cc --- /dev/null +++ b/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=with_forced_key.json @@ -0,0 +1,7 @@ +{ + "alg": "HS256", + "k": "Y2hhbmdlbWVjaGFuZ2VtZWNoYW5nZW1lY2hhbmdlbWU", + "kid": "7d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8", + "kty": "oct", + "use": "sig" +} diff --git a/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=without_cache.json b/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=without_cache.json new file mode 100644 index 00000000..f81e76cc --- /dev/null +++ b/jwksx/.snapshots/TestFetcherNext-case=resolve_single_source_url-case=without_cache.json @@ -0,0 +1,7 @@ +{ + "alg": "HS256", + "k": "Y2hhbmdlbWVjaGFuZ2VtZWNoYW5nZW1lY2hhbmdlbWU", + "kid": "7d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8", + "kty": "oct", + "use": "sig" +} diff --git a/jwksx/fetcher.go b/jwksx/fetcher.go index 01958e1d..4b476a30 100644 --- a/jwksx/fetcher.go +++ b/jwksx/fetcher.go @@ -13,6 +13,8 @@ import ( ) // Fetcher is a small helper for fetching JSON Web Keys from remote endpoints. +// +// DEPRECATED: Use FetcherNext instead. type Fetcher struct { sync.RWMutex remote string @@ -21,6 +23,8 @@ type Fetcher struct { } // NewFetcher returns a new fetcher that can download JSON Web Keys from remote endpoints. +// +// DEPRECATED: Use FetcherNext instead. func NewFetcher(remote string) *Fetcher { return &Fetcher{ remote: remote, @@ -30,6 +34,8 @@ func NewFetcher(remote string) *Fetcher { } // GetKey retrieves a JSON Web Key from the cache, fetches it from a remote if it is not yet cached or returns an error. +// +// DEPRECATED: Use FetcherNext instead. func (f *Fetcher) GetKey(kid string) (*jose.JSONWebKey, error) { f.RLock() if k, ok := f.keys[kid]; ok { diff --git a/jwksx/fetcher_v2.go b/jwksx/fetcher_v2.go new file mode 100644 index 00000000..09b840c6 --- /dev/null +++ b/jwksx/fetcher_v2.go @@ -0,0 +1,167 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package jwksx + +import ( + "context" + "crypto/sha256" + "time" + + "github.com/hashicorp/go-retryablehttp" + + "github.com/ory/x/fetcher" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/ory/x/otelx" + + "github.com/dgraph-io/ristretto" + "github.com/lestrrat-go/jwx/jwk" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +var ErrUnableToFindKeyID = errors.New("specified JWK kid can not be found in the JWK sets") + +type ( + fetcherNextOptions struct { + forceKID string + cacheTTL time.Duration + useCache bool + httpClient *retryablehttp.Client + } + // FetcherNext is a JWK fetcher that can be used to fetch JWKs from multiple locations. + FetcherNext struct { + cache *ristretto.Cache + } + // FetcherNextOption is a functional option for the FetcherNext. + FetcherNextOption func(*fetcherNextOptions) +) + +// NewFetcherNext returns a new FetcherNext instance. +func NewFetcherNext(cache *ristretto.Cache) *FetcherNext { + return &FetcherNext{ + cache: cache, + } +} + +// WithForceKID forces the key ID to be used. Required when multiple JWK sets are configured. +func WithForceKID(kid string) FetcherNextOption { + return func(o *fetcherNextOptions) { + o.forceKID = kid + } +} + +// WithCacheTTL sets the cache TTL. If not set, the TTL is unlimited. +func WithCacheTTL(ttl time.Duration) FetcherNextOption { + return func(o *fetcherNextOptions) { + o.cacheTTL = ttl + } +} + +// WithCacheEnabled enables the cache. +func WithCacheEnabled() FetcherNextOption { + return func(o *fetcherNextOptions) { + o.useCache = true + } +} + +// WithHTTPClient will use the given HTTP client to fetch the JSON Web Keys. +func WithHTTPClient(c *retryablehttp.Client) FetcherNextOption { + return func(o *fetcherNextOptions) { + o.httpClient = c + } +} + +func (f *FetcherNext) ResolveKey(ctx context.Context, locations string, modifiers ...FetcherNextOption) (jwk.Key, error) { + return f.ResolveKeyFromLocations(ctx, []string{locations}, modifiers...) +} + +func (f *FetcherNext) ResolveKeyFromLocations(ctx context.Context, locations []string, modifiers ...FetcherNextOption) (jwk.Key, error) { + opts := new(fetcherNextOptions) + for _, m := range modifiers { + m(opts) + } + + if len(locations) > 1 && opts.forceKID == "" { + return nil, errors.Errorf("a key ID must be specified when multiple JWK sets are configured") + } + + set := jwk.NewSet() + eg := new(errgroup.Group) + for k := range locations { + location := locations[k] + eg.Go(func() error { + remoteSet, err := f.fetch(ctx, location, opts) + if err != nil { + return err + } + + iterator := remoteSet.Iterate(ctx) + for iterator.Next(ctx) { + // Pair().Value is always of type jwk.Key when generated by Iterate. + set.Add(iterator.Pair().Value.(jwk.Key)) + } + + return nil + }) + } + + if err := eg.Wait(); err != nil { + return nil, err + } + + if opts.forceKID != "" { + key, found := set.LookupKeyID(opts.forceKID) + if !found { + return nil, errors.WithStack(ErrUnableToFindKeyID) + } + + return key, nil + } + + // No KID was forced? Use the first key we can find. + key, found := set.Get(0) + if !found { + return nil, errors.WithStack(ErrUnableToFindKeyID) + } + + return key, nil +} + +// fetch fetches the JWK set from the given location and if enabled, may use the cache to look up the JWK set. +func (f *FetcherNext) fetch(ctx context.Context, location string, opts *fetcherNextOptions) (_ jwk.Set, err error) { + tracer := trace.SpanFromContext(ctx).TracerProvider().Tracer("") + ctx, span := tracer.Start(ctx, "jwksx.FetcherNext.fetch", trace.WithAttributes(attribute.String("location", location))) + defer otelx.End(span, &err) + + cacheKey := sha256.Sum256([]byte(location)) + if opts.useCache { + if result, found := f.cache.Get(cacheKey[:]); found { + return result.(jwk.Set), nil + } + } + + var fopts []fetcher.Modifier + if opts.httpClient != nil { + fopts = append(fopts, fetcher.WithClient(opts.httpClient)) + } + + result, err := fetcher.NewFetcher(fopts...).FetchContext(ctx, location) + if err != nil { + return nil, err + } + + set, err := jwk.ParseReader(result) + if err != nil { + return nil, err + } + + if opts.useCache { + f.cache.SetWithTTL(cacheKey[:], set, 1, opts.cacheTTL) + } + + return set, nil +} diff --git a/jwksx/fetcher_v2_test.go b/jwksx/fetcher_v2_test.go new file mode 100644 index 00000000..6bf23f02 --- /dev/null +++ b/jwksx/fetcher_v2_test.go @@ -0,0 +1,208 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package jwksx + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hashicorp/go-retryablehttp" + "github.com/pkg/errors" + + "github.com/dgraph-io/ristretto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ory/x/snapshotx" +) + +const ( + multiKeys = `{ + "keys": [ + { + "use": "sig", + "kty": "oct", + "kid": "7d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8", + "alg": "HS256", + "k": "Y2hhbmdlbWVjaGFuZ2VtZWNoYW5nZW1lY2hhbmdlbWU" + }, + { + "use": "sig", + "kty": "oct", + "kid": "8d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8", + "alg": "HS256", + "k": "Y2hhbmdlbWVjaGFuZ2VtZWNoYW5nZW1lY2hhbmdlbWU" + }, + { + "use": "sig", + "kty": "oct", + "kid": "9d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8", + "alg": "HS256", + "k": "Y2hhbmdlbWVjaGFuZ2VtZWNoYW5nZW1lY2hhbmdlbWU" + } + ] +}` +) + +type brokenTransport struct{} + +var _ http.RoundTripper = new(brokenTransport) +var errBroken = errors.New("broken") + +func (b brokenTransport) RoundTrip(_ *http.Request) (*http.Response, error) { + return nil, errBroken +} + +func TestFetcherNext(t *testing.T) { + ctx := context.Background() + cache, _ := ristretto.NewCache(&ristretto.Config{ + NumCounters: 100 * 10, + MaxCost: 100, + BufferItems: 64, + Metrics: true, + IgnoreInternalCost: true, + Cost: func(value interface{}) int64 { + return 1 + }, + }) + f := NewFetcherNext(cache) + + createRemoteProvider := func(called *int, payload string) *httptest.Server { + cache.Clear() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *called++ + _, _ = w.Write([]byte(payload)) + })) + t.Cleanup(ts.Close) + return ts + } + + t.Run("case=resolve multiple source urls", func(t *testing.T) { + t.Run("case=fails without forced kid", func(t *testing.T) { + var called int + ts1 := createRemoteProvider(&called, keys) + ts2 := createRemoteProvider(&called, multiKeys) + + _, err := f.ResolveKeyFromLocations(ctx, []string{ts1.URL, ts2.URL}) + require.Error(t, err) + }) + t.Run("case=succeeds with forced kid", func(t *testing.T) { + var called int + ts1 := createRemoteProvider(&called, keys) + ts2 := createRemoteProvider(&called, multiKeys) + + k, err := f.ResolveKeyFromLocations(ctx, []string{ts1.URL, ts2.URL}, WithForceKID("8d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8")) + require.NoError(t, err) + snapshotx.SnapshotT(t, k) + }) + }) + t.Run("case=resolve single source url", func(t *testing.T) { + t.Run("case=with forced key", func(t *testing.T) { + var called int + ts := createRemoteProvider(&called, keys) + + k, err := f.ResolveKey(ctx, ts.URL, WithForceKID("7d5f5ad0674ec2f2960b1a34f33370a0f71471fa0e3ef0c0a692977d276dafe8")) + require.NoError(t, err) + snapshotx.SnapshotT(t, k) + }) + + t.Run("case=forced key is not found", func(t *testing.T) { + var called int + ts := createRemoteProvider(&called, keys) + + _, err := f.ResolveKey(ctx, ts.URL, WithForceKID("not-found")) + require.Error(t, err) + }) + + t.Run("case=no key in remote", func(t *testing.T) { + var called int + ts := createRemoteProvider(&called, "{}") + + _, err := f.ResolveKey(ctx, ts.URL) + require.Error(t, err) + }) + + t.Run("case=remote not returning JSON", func(t *testing.T) { + var called int + ts := createRemoteProvider(&called, "lol") + + _, err := f.ResolveKey(ctx, ts.URL) + require.Error(t, err) + }) + + t.Run("case=without cache", func(t *testing.T) { + var called int + ts := createRemoteProvider(&called, keys) + + k, err := f.ResolveKey(ctx, ts.URL) + require.NoError(t, err) + snapshotx.SnapshotT(t, k) + assert.Equal(t, called, 1) + + cache.Wait() + + _, err = f.ResolveKey(ctx, ts.URL) + require.NoError(t, err) + assert.Equal(t, called, 2) + }) + + t.Run("case=with cache", func(t *testing.T) { + var called int + ts := createRemoteProvider(&called, keys) + + k, err := f.ResolveKey(ctx, ts.URL, WithCacheEnabled()) + require.NoError(t, err) + assert.Equal(t, called, 1) + + cache.Wait() + + k, err = f.ResolveKey(ctx, ts.URL, WithCacheEnabled()) + require.NoError(t, err) + assert.Equal(t, called, 1) + + snapshotx.SnapshotT(t, k) + }) + + t.Run("case=with cache and TTL", func(t *testing.T) { + var called int + ts := createRemoteProvider(&called, keys) + waitTime := time.Millisecond * 100 + + k, err := f.ResolveKey(ctx, ts.URL, WithCacheEnabled(), WithCacheTTL(waitTime)) + require.NoError(t, err) + assert.Equal(t, called, 1) + + cache.Wait() + + k, err = f.ResolveKey(ctx, ts.URL, WithCacheEnabled()) + require.NoError(t, err) + assert.Equal(t, called, 1) + + time.Sleep(waitTime) + + cache.Wait() + + k, err = f.ResolveKey(ctx, ts.URL, WithCacheEnabled()) + require.NoError(t, err) + assert.Equal(t, called, 2) + + snapshotx.SnapshotT(t, k) + }) + + t.Run("case=with broken HTTP client", func(t *testing.T) { + var called int + ts := createRemoteProvider(&called, keys) + + broken := retryablehttp.NewClient() + broken.RetryMax = 0 + broken.HTTPClient.Transport = new(brokenTransport) + + _, err := f.ResolveKey(ctx, ts.URL, WithHTTPClient(broken)) + require.ErrorIs(t, err, errBroken) + }) + }) +} From d1b24d42515e898ebed22ba2cf92e47bf5c73e7a Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:11:23 +0300 Subject: [PATCH 27/56] fix: add missing pagination parameters to migrationpagination package (#721) --- pagination/migrationpagination/header.go | 31 ++++++++++++++++++++++-- pagination/pagepagination/header.go | 8 +++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pagination/migrationpagination/header.go b/pagination/migrationpagination/header.go index 0a24143c..47663ced 100644 --- a/pagination/migrationpagination/header.go +++ b/pagination/migrationpagination/header.go @@ -5,7 +5,9 @@ package migrationpagination // swagger:model mixedPaginationRequestParameters type RequestParameters struct { - // Items per Page + // Deprecated Items per Page + // + // DEPRECATED: Please use `page_token` instead. This parameter will be removed in the future. // // This is the number of items per page. // @@ -16,7 +18,9 @@ type RequestParameters struct { // max: 1000 PerPage int `json:"per_page"` - // Pagination Page + // Deprecated Pagination Page + // + // DEPRECATED: Please use `page_token` instead. This parameter will be removed in the future. // // This value is currently an integer, but it is not sequential. The value is not the page number, but a // reference. The next page can be any number and some numbers might return an empty list. @@ -28,6 +32,29 @@ type RequestParameters struct { // required: false // in: query Page int `json:"page"` + + // Page Size + // + // This is the number of items per page to return. For details on pagination please head over to the + // [pagination documentation](https://www.ory.sh/docs/ecosystem/api-design#pagination). + // + // required: false + // in: query + // default: 250 + // min: 1 + // max: 500 + PageSize int `json:"page_size"` + + // Next Page Token + // + // The next page token. For details on pagination please head over to the + // [pagination documentation](https://www.ory.sh/docs/ecosystem/api-design#pagination). + // + // required: false + // in: query + // default: 1 + // min: 1 + PageToken string `json:"page_token"` } // Pagination Response Header diff --git a/pagination/pagepagination/header.go b/pagination/pagepagination/header.go index 302c0d29..64ab0653 100644 --- a/pagination/pagepagination/header.go +++ b/pagination/pagepagination/header.go @@ -35,8 +35,8 @@ type RequestParameters struct { // Items per Page // - // This is the number of items per page to return. - // For details on pagination please head over to the [pagination documentation](https://www.ory.sh/docs/ecosystem/api-design#pagination). + // This is the number of items per page to return. For details on pagination please head over to the + // [pagination documentation](https://www.ory.sh/docs/ecosystem/api-design#pagination). // // required: false // in: query @@ -47,8 +47,8 @@ type RequestParameters struct { // Next Page Token // - // The next page token. - // For details on pagination please head over to the [pagination documentation](https://www.ory.sh/docs/ecosystem/api-design#pagination). + // The next page token. For details on pagination please head over to the + // [pagination documentation](https://www.ory.sh/docs/ecosystem/api-design#pagination). // // required: false // in: query From fbf9d352180c54d6aa5dd57cf0c4ad8e487e223c Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:31:54 +0300 Subject: [PATCH 28/56] feat: add token pagination link header parser (#722) --- go.mod | 1 + go.sum | 2 + pagination/keysetpagination/parse_header.go | 44 +++++++++++++++++ .../keysetpagination/parse_header_test.go | 49 +++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 pagination/keysetpagination/parse_header.go create mode 100644 pagination/keysetpagination/parse_header_test.go diff --git a/go.mod b/go.mod index d31af759..4c6617e8 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,7 @@ require ( github.com/ory/herodot v0.9.13 github.com/ory/jsonschema/v3 v3.0.7 github.com/pelletier/go-toml v1.9.5 + github.com/peterhellberg/link v1.2.0 github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.7.0 github.com/prometheus/client_golang v1.13.0 diff --git a/go.sum b/go.sum index 6c964ec1..c60eaa8c 100644 --- a/go.sum +++ b/go.sum @@ -752,6 +752,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= +github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= diff --git a/pagination/keysetpagination/parse_header.go b/pagination/keysetpagination/parse_header.go new file mode 100644 index 00000000..8be68b03 --- /dev/null +++ b/pagination/keysetpagination/parse_header.go @@ -0,0 +1,44 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package keysetpagination + +import ( + "net/http" + "net/url" + + "github.com/peterhellberg/link" +) + +// PaginationResult represents a parsed result of the link HTTP header. +type PaginationResult struct { + // NextToken is the next page token. If it's empty, there is no next page. + NextToken string + + // FirstToken is the first page token. + FirstToken string +} + +// ParseHeader parses the response header's Link. +func ParseHeader(r *http.Response) *PaginationResult { + links := link.ParseResponse(r) + return &PaginationResult{ + NextToken: findRel(links, "next"), + FirstToken: findRel(links, "first"), + } +} + +func findRel(links link.Group, rel string) string { + for idx, l := range links { + if idx == rel { + parsed, err := url.Parse(l.URI) + if err != nil { + continue + } + + return parsed.Query().Get("page_token") + } + } + + return "" +} diff --git a/pagination/keysetpagination/parse_header_test.go b/pagination/keysetpagination/parse_header_test.go new file mode 100644 index 00000000..99ade8ae --- /dev/null +++ b/pagination/keysetpagination/parse_header_test.go @@ -0,0 +1,49 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package keysetpagination + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseHeader(t *testing.T) { + u, err := url.Parse("https://www.ory.sh/") + require.NoError(t, err) + + t.Run("has next page", func(t *testing.T) { + p := &Paginator{ + defaultToken: StringPageToken("default"), + token: StringPageToken("next"), + size: 2, + } + + r := httptest.NewRecorder() + Header(r, u, p) + + result := ParseHeader(&http.Response{Header: r.Header()}) + assert.Equal(t, "next", result.NextToken, r.Header()) + assert.Equal(t, "default", result.FirstToken, r.Header()) + }) + + t.Run("is last page", func(t *testing.T) { + p := &Paginator{ + defaultToken: StringPageToken("default"), + size: 1, + isLast: true, + } + + r := httptest.NewRecorder() + Header(r, u, p) + + result := ParseHeader(&http.Response{Header: r.Header()}) + assert.Equal(t, "", result.NextToken, r.Header()) + assert.Equal(t, "default", result.FirstToken, r.Header()) + }) +} From 78da4d72b89b59b4d242cd728608fbf6b5ebe7bb Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Thu, 14 Sep 2023 13:19:09 +0200 Subject: [PATCH 29/56] feat: add geolocation to trace events (#724) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Miłosz Szekiel <12242002+mszekiel@users.noreply.github.com> --- httpx/client_info.go | 14 ++++++++++++++ httpx/client_info_test.go | 32 ++++++++++++++++++++++++++++++++ otelx/semconv/context.go | 9 ++++++++- otelx/semconv/context_test.go | 14 ++++++++++++-- otelx/semconv/events.go | 27 ++++++++++++++++++++++++--- 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/httpx/client_info.go b/httpx/client_info.go index 8d6c3a98..14f915d5 100644 --- a/httpx/client_info.go +++ b/httpx/client_info.go @@ -9,6 +9,12 @@ import ( "strings" ) +type GeoLocation struct { + City string + Region string + Country string +} + func GetClientIPAddressesWithoutInternalIPs(ipAddresses []string) (string, error) { var res string @@ -36,3 +42,11 @@ func ClientIP(r *http.Request) string { return r.RemoteAddr } } + +func ClientGeoLocation(r *http.Request) *GeoLocation { + return &GeoLocation{ + City: r.Header.Get("Cf-Ipcity"), + Region: r.Header.Get("Cf-Region-Code"), + Country: r.Header.Get("Cf-Ipcountry"), + } +} diff --git a/httpx/client_info_test.go b/httpx/client_info_test.go index f3682f4d..b7a1dba9 100644 --- a/httpx/client_info_test.go +++ b/httpx/client_info_test.go @@ -58,3 +58,35 @@ func TestClientIP(t *testing.T) { assert.Equal(t, "1.0.0.4", ClientIP(req)) }) } + +func TestClientGeoLocation(t *testing.T) { + req := http.Request{ + Header: http.Header{}, + } + req.Header.Add("cf-ipcity", "Berlin") + req.Header.Add("cf-ipcountry", "Germany") + req.Header.Add("cf-region-code", "BE") + + t.Run("cf-ipcity", func(t *testing.T) { + req := req.Clone(context.Background()) + assert.Equal(t, "Berlin", ClientGeoLocation(req).City) + }) + + t.Run("cf-ipcountry", func(t *testing.T) { + req := req.Clone(context.Background()) + assert.Equal(t, "Germany", ClientGeoLocation(req).Country) + }) + + t.Run("cf-region-code", func(t *testing.T) { + req := req.Clone(context.Background()) + assert.Equal(t, "BE", ClientGeoLocation(req).Region) + }) + + t.Run("empty", func(t *testing.T) { + req := req.Clone(context.Background()) + req.Header.Del("cf-ipcity") + req.Header.Del("cf-ipcountry") + req.Header.Del("cf-region-code") + assert.Equal(t, GeoLocation{}, *ClientGeoLocation(req)) + }) +} diff --git a/otelx/semconv/context.go b/otelx/semconv/context.go index 07c0768b..a67bfd42 100644 --- a/otelx/semconv/context.go +++ b/otelx/semconv/context.go @@ -36,7 +36,14 @@ func AttributesFromContext(ctx context.Context) []attribute.KeyValue { } func Middleware(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - next(rw, r.WithContext(ContextWithAttributes(r.Context(), AttrClientIP(httpx.ClientIP(r))))) + ctx := ContextWithAttributes(r.Context(), + append( + AttrGeoLocation(*httpx.ClientGeoLocation(r)), + AttrClientIP(httpx.ClientIP(r)), + )..., + ) + + next(rw, r.WithContext(ctx)) } func reverse[S ~[]E, E any](s S) { diff --git a/otelx/semconv/context_test.go b/otelx/semconv/context_test.go index e0a7b311..a68882fb 100644 --- a/otelx/semconv/context_test.go +++ b/otelx/semconv/context_test.go @@ -10,6 +10,8 @@ import ( "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" "go.opentelemetry.io/otel/attribute" + + "github.com/ory/x/httpx" ) func TestAttributesFromContext(t *testing.T) { @@ -21,11 +23,19 @@ func TestAttributesFromContext(t *testing.T) { assert.Len(t, AttributesFromContext(ctx), 1) uid1, uid2 := uuid.Must(uuid.NewV4()), uuid.Must(uuid.NewV4()) - ctx = ContextWithAttributes(ctx, AttrIdentityID(uid1), AttrClientIP("127.0.0.1"), AttrIdentityID(uid2)) + location := httpx.GeoLocation{ + City: "Berlin", + Country: "Germany", + Region: "BE", + } + ctx = ContextWithAttributes(ctx, append(AttrGeoLocation(location), AttrIdentityID(uid1), AttrClientIP("127.0.0.1"), AttrIdentityID(uid2))...) attrs := AttributesFromContext(ctx) - assert.Len(t, attrs, 3, "should deduplicate") + assert.Len(t, attrs, 6, "should deduplicate") assert.Equal(t, []attribute.KeyValue{ attribute.String(AttributeKeyNID.String(), nid.String()), + attribute.String(AttributeKeyGeoLocationCity.String(), "Berlin"), + attribute.String(AttributeKeyGeoLocationCountry.String(), "Germany"), + attribute.String(AttributeKeyGeoLocationRegion.String(), "BE"), attribute.String(AttributeKeyClientIP.String(), "127.0.0.1"), attribute.String(AttributeKeyIdentityID.String(), uid2.String()), }, attrs, "last duplicate attribute wins") diff --git a/otelx/semconv/events.go b/otelx/semconv/events.go index cece6fb6..05e5e4b9 100644 --- a/otelx/semconv/events.go +++ b/otelx/semconv/events.go @@ -7,6 +7,8 @@ package semconv import ( "github.com/gofrs/uuid" otelattr "go.opentelemetry.io/otel/attribute" + + "github.com/ory/x/httpx" ) type Event string @@ -22,9 +24,12 @@ func (a AttributeKey) String() string { } const ( - AttributeKeyIdentityID AttributeKey = "IdentityID" - AttributeKeyNID AttributeKey = "ProjectID" - AttributeKeyClientIP AttributeKey = "ClientIP" + AttributeKeyIdentityID AttributeKey = "IdentityID" + AttributeKeyNID AttributeKey = "ProjectID" + AttributeKeyClientIP AttributeKey = "ClientIP" + AttributeKeyGeoLocationCity AttributeKey = "GeoLocationCity" + AttributeKeyGeoLocationRegion AttributeKey = "GeoLocationRegion" + AttributeKeyGeoLocationCountry AttributeKey = "GeoLocationCountry" ) func AttrIdentityID(val uuid.UUID) otelattr.KeyValue { @@ -38,3 +43,19 @@ func AttrNID(val uuid.UUID) otelattr.KeyValue { func AttrClientIP(val string) otelattr.KeyValue { return otelattr.String(AttributeKeyClientIP.String(), val) } + +func AttrGeoLocation(val httpx.GeoLocation) []otelattr.KeyValue { + geoLocationAttributes := make([]otelattr.KeyValue, 0, 3) + + if val.City != "" { + geoLocationAttributes = append(geoLocationAttributes, otelattr.String(AttributeKeyGeoLocationCity.String(), val.City)) + } + if val.Country != "" { + geoLocationAttributes = append(geoLocationAttributes, otelattr.String(AttributeKeyGeoLocationCountry.String(), val.Country)) + } + if val.Region != "" { + geoLocationAttributes = append(geoLocationAttributes, otelattr.String(AttributeKeyGeoLocationRegion.String(), val.Region)) + } + + return geoLocationAttributes +} From 45f6c9069c471d1f78f2413c9f9e6547b77166bd Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Fri, 6 Oct 2023 10:59:24 +0200 Subject: [PATCH 30/56] feat: force clientFoundRows to true (#727) * feat: force clientFoundRows to true * fix: adjust golangci lint * test: update assertion --- .github/workflows/test.yml | 2 +- Makefile | 2 +- sqlcon/parse_opts.go | 4 ++++ sqlcon/parse_opts_test.go | 6 +++--- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4933ee29..51fcda13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: skip-go-installation: true - args: --timeout 2m + args: --timeout 5m - name: Install cockroach DB run: | curl https://binaries.cockroachdb.com/cockroach-v22.2.5.linux-amd64.tgz | tar -xz diff --git a/Makefile b/Makefile index 7e03f305..671b3b1b 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ licenses: .bin/licenses node_modules # checks open-source licenses GOBIN=$(shell pwd)/.bin go install golang.org/x/tools/cmd/goimports@latest .bin/golangci-lint: Makefile - bash <(curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh) -d -b .bin v1.46.2 + bash <(curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh) -d -b .bin v1.54.2 .bin/licenses: Makefile curl https://raw.githubusercontent.com/ory/ci/master/licenses/install | sh diff --git a/sqlcon/parse_opts.go b/sqlcon/parse_opts.go index 0c47fd6d..f25c310a 100644 --- a/sqlcon/parse_opts.go +++ b/sqlcon/parse_opts.go @@ -109,6 +109,10 @@ func FinalizeDSN(l *logrusx.Logger, dsn string) string { q.Set("multiStatements", "true") q.Set("parseTime", "true") + // Thius causes an UPDATE to return the number of matching rows instead of + // the number of rows changed. This ensures compatibility with PostgreSQL and SQLite behavior. + q.Set("clientFoundRows", "true") + return fmt.Sprintf("%s?%s", parts[0], q.Encode()) } diff --git a/sqlcon/parse_opts_test.go b/sqlcon/parse_opts_test.go index dfcff013..0e24cae1 100644 --- a/sqlcon/parse_opts_test.go +++ b/sqlcon/parse_opts_test.go @@ -102,11 +102,11 @@ func TestFinalizeDSN(t *testing.T) { }{ { dsn: "mysql://localhost", - expected: "mysql://localhost?multiStatements=true&parseTime=true", + expected: "mysql://localhost?clientFoundRows=true&multiStatements=true&parseTime=true", }, { - dsn: "mysql://localhost?multiStatements=true&parseTime=true", - expected: "mysql://localhost?multiStatements=true&parseTime=true", + dsn: "mysql://localhost?multiStatements=true&parseTime=true&clientFoundRows=false", + expected: "mysql://localhost?clientFoundRows=true&multiStatements=true&parseTime=true", }, { dsn: "postgres://localhost", From 67f2a27a943d89ad9392a7803343513c4dcb0865 Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Tue, 10 Oct 2023 10:45:57 +0200 Subject: [PATCH 31/56] fix: link header in keyset pagination (#729) --- pagination/keysetpagination/header.go | 11 ++++----- pagination/keysetpagination/header_test.go | 26 +++++++++++++--------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/pagination/keysetpagination/header.go b/pagination/keysetpagination/header.go index b7474e0c..e523859b 100644 --- a/pagination/keysetpagination/header.go +++ b/pagination/keysetpagination/header.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "github.com/pkg/errors" ) @@ -83,11 +84,11 @@ func header(u *url.URL, rel, token string, size int) string { // It contains links to the first and next page, if one exists. func Header(w http.ResponseWriter, u *url.URL, p *Paginator) { size := p.Size() - w.Header().Set("Link", header(u, "first", p.defaultToken.Encode(), size)) - - if !p.IsLast() { - w.Header().Add("Link", header(u, "next", p.Token().Encode(), size)) + link := []string{header(u, "first", p.defaultToken.Encode(), size)} + if !p.isLast { + link = append(link, header(u, "next", p.Token().Encode(), size)) } + w.Header().Set("Link", strings.Join(link, ",")) } // Parse returns the pagination options from the URL query. @@ -104,7 +105,7 @@ func Parse(q url.Values, p PageTokenConstructor) ([]Option, error) { } opts = append(opts, WithToken(parsed)) } - if q.Has("page_size") { + if q.Get("page_size") != "" { size, err := strconv.Atoi(q.Get("page_size")) if err != nil { return nil, errors.WithStack(err) diff --git a/pagination/keysetpagination/header_test.go b/pagination/keysetpagination/header_test.go index 19186f4e..a6eb20e3 100644 --- a/pagination/keysetpagination/header_test.go +++ b/pagination/keysetpagination/header_test.go @@ -8,6 +8,7 @@ import ( "net/url" "testing" + "github.com/peterhellberg/link" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,19 +27,22 @@ func TestHeader(t *testing.T) { Header(r, u, p) - links := r.HeaderMap["Link"] - require.Len(t, links, 2) - assert.Contains(t, links[0], "page_token=default") - assert.Contains(t, links[1], "page_token=next") + assert.Len(t, r.Result().Header.Values("link"), 1, "make sure we send one header with multiple comma-separated values rather than multiple headers") - t.Run("with isLast", func(t *testing.T) { - p.isLast = true + links := link.ParseResponse(r.Result()) + assert.Contains(t, links, "first") + assert.Contains(t, links["first"].URI, "page_token=default") - Header(r, u, p) + assert.Contains(t, links, "next") + assert.Contains(t, links["next"].URI, "page_token=next") - links := r.HeaderMap["Link"] - require.Len(t, links, 1) - assert.Contains(t, links[0], "page_token=default") - }) + p.isLast = true + r = httptest.NewRecorder() + Header(r, u, p) + links = link.ParseResponse(r.Result()) + + assert.Contains(t, links, "first") + assert.Contains(t, links["first"].URI, "page_token=default") + assert.NotContains(t, links, "next") } From 4d5bf8d7968b2a1afa59307c5ce6f1d9c29f4aa1 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Wed, 11 Oct 2023 12:13:10 +0200 Subject: [PATCH 32/56] feat: add crdb consistency guarantees (#728) --- crdbx/staleness.go | 105 ++++++++++++++++++++++++++++++++++++++++ crdbx/staleness_test.go | 73 ++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 crdbx/staleness.go create mode 100644 crdbx/staleness_test.go diff --git a/crdbx/staleness.go b/crdbx/staleness.go new file mode 100644 index 00000000..b3b8fabc --- /dev/null +++ b/crdbx/staleness.go @@ -0,0 +1,105 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package crdbx + +import ( + "net/http" + + "github.com/ory/x/dbal" + + "github.com/gobuffalo/pop/v6" + + "github.com/ory/x/sqlcon" +) + +// Control API consistency guarantees +// +// swagger:model consistencyRequestParameters +type ConsistencyRequestParameters struct { + // Read Consistency Level (experimental) + // + // The read consistency level determines the consistency guarantee for reads and queries: + // + // - strong (slow): The read is guaranteed to return the most recent data committed at the start of the read. + // - eventual (very fast): The result will return data that is about 4.8 seconds old. + // + // The default consistency guarantee can be changed in the Ory Network Console or using the Ory CLI with + // `ory patch project --replace '/serve/default_consistency_level="strong"'`. + // + // This feature is fully functional only in Ory Network and currently experimental. + // + // required: false + // in: query + Consistency ConsistencyLevel `json:"consistency"` +} + +// ConsistencyLevel is the consistency level. +// swagger:enum ConsistencyLevel +type ConsistencyLevel string + +const ( + // ConsistencyLevelUnset is the unset / default consistency level. + ConsistencyLevelUnset ConsistencyLevel = "" + // ConsistencyLevelStrong is the strong consistency level. + ConsistencyLevelStrong ConsistencyLevel = "strong" + // ConsistencyLevelEventual is the eventual consistency level using follower read timestamps. + ConsistencyLevelEventual ConsistencyLevel = "eventual" +) + +// ConsistencyLevelFromRequest extracts the consistency level from a request. +func ConsistencyLevelFromRequest(r *http.Request) ConsistencyLevel { + return ConsistencyLevelFromString(r.URL.Query().Get("consistency")) +} + +// ConsistencyLevelFromString converts a string to a ConsistencyLevel. +// If the string is not recognized or unset, ConsistencyLevelStrong is returned. +func ConsistencyLevelFromString(in string) ConsistencyLevel { + switch in { + case string(ConsistencyLevelStrong): + return ConsistencyLevelStrong + case string(ConsistencyLevelEventual): + return ConsistencyLevelEventual + case string(ConsistencyLevelUnset): + return ConsistencyLevelStrong + } + return ConsistencyLevelStrong +} + +// SetTransactionConsistency sets the transaction consistency level for CockroachDB. +func SetTransactionConsistency(c *pop.Connection, level ConsistencyLevel, fallback ConsistencyLevel) error { + q := getTransactionConsistencyQuery(c.Dialect.Name(), level, fallback) + if len(q) == 0 { + return nil + } + + return sqlcon.HandleError(c.RawQuery(q).Exec()) +} + +const transactionFollowerReadTimestamp = "SET TRANSACTION AS OF SYSTEM TIME follower_read_timestamp()" + +func getTransactionConsistencyQuery(dialect string, level ConsistencyLevel, fallback ConsistencyLevel) string { + if dialect != dbal.DriverCockroachDB { + // Only CockroachDB supports this. + return "" + } + + switch level { + case ConsistencyLevelStrong: + // Nothing to do + return "" + case ConsistencyLevelEventual: + // Jumps to end of function + case ConsistencyLevelUnset: + fallthrough + default: + if fallback != ConsistencyLevelEventual { + // Nothing to do + return "" + } + + // Jumps to end of function + } + + return transactionFollowerReadTimestamp +} diff --git a/crdbx/staleness_test.go b/crdbx/staleness_test.go new file mode 100644 index 00000000..ef29416d --- /dev/null +++ b/crdbx/staleness_test.go @@ -0,0 +1,73 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package crdbx + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/ory/x/urlx" +) + +func TestConsistencyLevelFromString(t *testing.T) { + assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromString("")) + assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromString("strong")) + assert.Equal(t, ConsistencyLevelEventual, ConsistencyLevelFromString("eventual")) + assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromString("lol")) +} + +func TestConsistencyLevelFromRequest(t *testing.T) { + assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromRequest(&http.Request{URL: urlx.ParseOrPanic("/?consistency=strong")})) + assert.Equal(t, ConsistencyLevelEventual, ConsistencyLevelFromRequest(&http.Request{URL: urlx.ParseOrPanic("/?consistency=eventual")})) + assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromRequest(&http.Request{URL: urlx.ParseOrPanic("/?consistency=asdf")})) + +} + +func TestGetTransactionConsistency(t *testing.T) { + for k, tc := range []struct { + in ConsistencyLevel + fallback ConsistencyLevel + dialect string + expected string + }{ + { + in: ConsistencyLevelUnset, + fallback: ConsistencyLevelStrong, + dialect: "cockroach", + expected: "", + }, + { + in: ConsistencyLevelStrong, + fallback: ConsistencyLevelStrong, + dialect: "cockroach", + expected: "", + }, + { + in: ConsistencyLevelStrong, + fallback: ConsistencyLevelEventual, + dialect: "cockroach", + expected: "", + }, + { + in: ConsistencyLevelUnset, + fallback: ConsistencyLevelEventual, + dialect: "cockroach", + expected: transactionFollowerReadTimestamp, + }, + { + in: ConsistencyLevelEventual, + fallback: ConsistencyLevelEventual, + dialect: "cockroach", + expected: transactionFollowerReadTimestamp, + }, + } { + t.Run(fmt.Sprintf("case=%d", k), func(t *testing.T) { + q := getTransactionConsistencyQuery(tc.dialect, tc.in, tc.fallback) + assert.EqualValues(t, tc.expected, q) + }) + } +} From 53c3843f806255ced18bded0d7cc8808e74291d3 Mon Sep 17 00:00:00 2001 From: aeneasr <3372410+aeneasr@users.noreply.github.com> Date: Wed, 11 Oct 2023 12:46:59 +0200 Subject: [PATCH 33/56] docs: update docstring for consistency --- crdbx/staleness.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crdbx/staleness.go b/crdbx/staleness.go index b3b8fabc..7fb5423a 100644 --- a/crdbx/staleness.go +++ b/crdbx/staleness.go @@ -17,17 +17,22 @@ import ( // // swagger:model consistencyRequestParameters type ConsistencyRequestParameters struct { - // Read Consistency Level (experimental) + // Read Consistency Level (preview) // - // The read consistency level determines the consistency guarantee for reads and queries: + // The read consistency level determines the consistency guarantee for reads: // // - strong (slow): The read is guaranteed to return the most recent data committed at the start of the read. // - eventual (very fast): The result will return data that is about 4.8 seconds old. // // The default consistency guarantee can be changed in the Ory Network Console or using the Ory CLI with - // `ory patch project --replace '/serve/default_consistency_level="strong"'`. + // `ory patch project --replace '/previews/default_read_consistency_level="strong"'`. // - // This feature is fully functional only in Ory Network and currently experimental. + // Setting the default consistency level to `eventual` may cause regressions in the future as we add consistency + // controls to more APIs. Currently, the following APIs will be affected by this setting: + // + // - `GET /admin/identities` + // + // This feature is in preview and only available in Ory Network. // // required: false // in: query From 4126131c916ef5e87e192c64628ff8a1185923a6 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:36:29 +0200 Subject: [PATCH 34/56] fix: correctly return unset for consistency (#732) --- crdbx/staleness.go | 2 +- crdbx/staleness_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crdbx/staleness.go b/crdbx/staleness.go index 7fb5423a..86ed0d9f 100644 --- a/crdbx/staleness.go +++ b/crdbx/staleness.go @@ -66,7 +66,7 @@ func ConsistencyLevelFromString(in string) ConsistencyLevel { case string(ConsistencyLevelEventual): return ConsistencyLevelEventual case string(ConsistencyLevelUnset): - return ConsistencyLevelStrong + return ConsistencyLevelUnset } return ConsistencyLevelStrong } diff --git a/crdbx/staleness_test.go b/crdbx/staleness_test.go index ef29416d..ae9dc099 100644 --- a/crdbx/staleness_test.go +++ b/crdbx/staleness_test.go @@ -14,7 +14,7 @@ import ( ) func TestConsistencyLevelFromString(t *testing.T) { - assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromString("")) + assert.Equal(t, ConsistencyLevelUnset, ConsistencyLevelFromString("")) assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromString("strong")) assert.Equal(t, ConsistencyLevelEventual, ConsistencyLevelFromString("eventual")) assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromString("lol")) @@ -24,6 +24,7 @@ func TestConsistencyLevelFromRequest(t *testing.T) { assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromRequest(&http.Request{URL: urlx.ParseOrPanic("/?consistency=strong")})) assert.Equal(t, ConsistencyLevelEventual, ConsistencyLevelFromRequest(&http.Request{URL: urlx.ParseOrPanic("/?consistency=eventual")})) assert.Equal(t, ConsistencyLevelStrong, ConsistencyLevelFromRequest(&http.Request{URL: urlx.ParseOrPanic("/?consistency=asdf")})) + assert.Equal(t, ConsistencyLevelUnset, ConsistencyLevelFromRequest(&http.Request{URL: urlx.ParseOrPanic("/?consistency")})) } From 0e34bc42e06d97568e021d3b80e7c31a8365e859 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:40:09 +0200 Subject: [PATCH 35/56] feat: add a read only transaction helper (#734) --- crdbx/readonly.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 crdbx/readonly.go diff --git a/crdbx/readonly.go b/crdbx/readonly.go new file mode 100644 index 00000000..d8f67769 --- /dev/null +++ b/crdbx/readonly.go @@ -0,0 +1,21 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package crdbx + +import ( + "github.com/gobuffalo/pop/v6" + + "github.com/ory/x/dbal" + "github.com/ory/x/sqlcon" +) + +// SetTransactionReadOnly sets the transaction to read only for CockroachDB. +func SetTransactionReadOnly(c *pop.Connection) error { + if c.Dialect.Name() != dbal.DriverCockroachDB { + // Only CockroachDB supports this. + return nil + } + + return sqlcon.HandleError(c.RawQuery("SET TRANSACTION READ ONLY").Exec()) +} From e6c61ed41fc506d1a0bc9fc617bc53948b0addf6 Mon Sep 17 00:00:00 2001 From: Patrik Date: Thu, 12 Oct 2023 15:45:20 +0200 Subject: [PATCH 36/56] feat(otelx): add workspace attribute (#735) --- otelx/semconv/context_test.go | 9 +++++---- otelx/semconv/events.go | 5 +++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/otelx/semconv/context_test.go b/otelx/semconv/context_test.go index a68882fb..a1ea9f49 100644 --- a/otelx/semconv/context_test.go +++ b/otelx/semconv/context_test.go @@ -18,9 +18,9 @@ func TestAttributesFromContext(t *testing.T) { ctx := context.Background() assert.Len(t, AttributesFromContext(ctx), 0) - nid := uuid.Must(uuid.NewV4()) - ctx = ContextWithAttributes(ctx, AttrNID(nid)) - assert.Len(t, AttributesFromContext(ctx), 1) + nid, wsID := uuid.Must(uuid.NewV4()), uuid.Must(uuid.NewV4()) + ctx = ContextWithAttributes(ctx, AttrNID(nid), AttrWorkspace(wsID)) + assert.Len(t, AttributesFromContext(ctx), 2) uid1, uid2 := uuid.Must(uuid.NewV4()), uuid.Must(uuid.NewV4()) location := httpx.GeoLocation{ @@ -30,9 +30,10 @@ func TestAttributesFromContext(t *testing.T) { } ctx = ContextWithAttributes(ctx, append(AttrGeoLocation(location), AttrIdentityID(uid1), AttrClientIP("127.0.0.1"), AttrIdentityID(uid2))...) attrs := AttributesFromContext(ctx) - assert.Len(t, attrs, 6, "should deduplicate") + assert.Len(t, attrs, 7, "should deduplicate") assert.Equal(t, []attribute.KeyValue{ attribute.String(AttributeKeyNID.String(), nid.String()), + attribute.String(AttributeKeyWorkspace.String(), wsID.String()), attribute.String(AttributeKeyGeoLocationCity.String(), "Berlin"), attribute.String(AttributeKeyGeoLocationCountry.String(), "Germany"), attribute.String(AttributeKeyGeoLocationRegion.String(), "BE"), diff --git a/otelx/semconv/events.go b/otelx/semconv/events.go index 05e5e4b9..c1739191 100644 --- a/otelx/semconv/events.go +++ b/otelx/semconv/events.go @@ -30,6 +30,7 @@ const ( AttributeKeyGeoLocationCity AttributeKey = "GeoLocationCity" AttributeKeyGeoLocationRegion AttributeKey = "GeoLocationRegion" AttributeKeyGeoLocationCountry AttributeKey = "GeoLocationCountry" + AttributeKeyWorkspace AttributeKey = "WorkspaceID" ) func AttrIdentityID(val uuid.UUID) otelattr.KeyValue { @@ -40,6 +41,10 @@ func AttrNID(val uuid.UUID) otelattr.KeyValue { return otelattr.String(AttributeKeyNID.String(), val.String()) } +func AttrWorkspace(val uuid.UUID) otelattr.KeyValue { + return otelattr.String(AttributeKeyWorkspace.String(), val.String()) +} + func AttrClientIP(val string) otelattr.KeyValue { return otelattr.String(AttributeKeyClientIP.String(), val) } From 771b19c35ca9105506c03eab34e788a9b629fea3 Mon Sep 17 00:00:00 2001 From: Ferdynand Naczynski Date: Thu, 26 Oct 2023 15:10:30 +0200 Subject: [PATCH 37/56] feat: add custom JWK parse error type (#739) --- jwksx/fetcher_v2.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwksx/fetcher_v2.go b/jwksx/fetcher_v2.go index 09b840c6..a0ea7590 100644 --- a/jwksx/fetcher_v2.go +++ b/jwksx/fetcher_v2.go @@ -8,6 +8,8 @@ import ( "crypto/sha256" "time" + "github.com/ory/herodot" + "github.com/hashicorp/go-retryablehttp" "github.com/ory/x/fetcher" @@ -156,7 +158,7 @@ func (f *FetcherNext) fetch(ctx context.Context, location string, opts *fetcherN set, err := jwk.ParseReader(result) if err != nil { - return nil, err + return nil, errors.WithStack(herodot.ErrBadRequest.WithReason("failed to parse JWK set").WithWrap(err)) } if opts.useCache { From acad8b084df5c3a47ed829856cabcdad826eaceb Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:24:53 +0100 Subject: [PATCH 38/56] feat: add batch insert (#744) --- ...t_buildInsertQueryArgs-case=cockroach.json | 14 + ...t_buildInsertQueryArgs-case=testModel.json | 14 + ...yValues-case=testModel-case=cockroach.json | 16 + sqlxx/batch/create.go | 275 ++++++++++++++++++ sqlxx/batch/create_test.go | 122 ++++++++ 5 files changed, 441 insertions(+) create mode 100644 sqlxx/batch/.snapshots/Test_buildInsertQueryArgs-case=cockroach.json create mode 100644 sqlxx/batch/.snapshots/Test_buildInsertQueryArgs-case=testModel.json create mode 100644 sqlxx/batch/.snapshots/Test_buildInsertQueryValues-case=testModel-case=cockroach.json create mode 100644 sqlxx/batch/create.go create mode 100644 sqlxx/batch/create_test.go diff --git a/sqlxx/batch/.snapshots/Test_buildInsertQueryArgs-case=cockroach.json b/sqlxx/batch/.snapshots/Test_buildInsertQueryArgs-case=cockroach.json new file mode 100644 index 00000000..51b3ae70 --- /dev/null +++ b/sqlxx/batch/.snapshots/Test_buildInsertQueryArgs-case=cockroach.json @@ -0,0 +1,14 @@ +{ + "TableName": "\"test_models\"", + "ColumnsDecl": "\"created_at\", \"id\", \"int\", \"nid\", \"null_time_ptr\", \"string\", \"updated_at\"", + "Columns": [ + "created_at", + "id", + "int", + "nid", + "null_time_ptr", + "string", + "updated_at" + ], + "Placeholders": "(?, ?, ?, ?, ?, ?, ?),\n(?, gen_random_uuid(), ?, ?, ?, ?, ?),\n(?, gen_random_uuid(), ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, gen_random_uuid(), ?, ?, ?, ?, ?),\n(?, gen_random_uuid(), ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, gen_random_uuid(), ?, ?, ?, ?, ?),\n(?, gen_random_uuid(), ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?)" +} diff --git a/sqlxx/batch/.snapshots/Test_buildInsertQueryArgs-case=testModel.json b/sqlxx/batch/.snapshots/Test_buildInsertQueryArgs-case=testModel.json new file mode 100644 index 00000000..db458b94 --- /dev/null +++ b/sqlxx/batch/.snapshots/Test_buildInsertQueryArgs-case=testModel.json @@ -0,0 +1,14 @@ +{ + "TableName": "\"test_models\"", + "ColumnsDecl": "\"created_at\", \"id\", \"int\", \"nid\", \"null_time_ptr\", \"string\", \"updated_at\"", + "Columns": [ + "created_at", + "id", + "int", + "nid", + "null_time_ptr", + "string", + "updated_at" + ], + "Placeholders": "(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?),\n(?, ?, ?, ?, ?, ?, ?)" +} diff --git a/sqlxx/batch/.snapshots/Test_buildInsertQueryValues-case=testModel-case=cockroach.json b/sqlxx/batch/.snapshots/Test_buildInsertQueryValues-case=testModel-case=cockroach.json new file mode 100644 index 00000000..c5bdc385 --- /dev/null +++ b/sqlxx/batch/.snapshots/Test_buildInsertQueryValues-case=testModel-case=cockroach.json @@ -0,0 +1,16 @@ +[ + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "string", + 42, + null, + { + "ID": "00000000-0000-0000-0000-000000000000", + "NID": "00000000-0000-0000-0000-000000000000", + "String": "string", + "Int": 42, + "NullTimePtr": null, + "created_at": "0001-01-01T00:00:00Z", + "updated_at": "0001-01-01T00:00:00Z" + } +] diff --git a/sqlxx/batch/create.go b/sqlxx/batch/create.go new file mode 100644 index 00000000..801dcbdd --- /dev/null +++ b/sqlxx/batch/create.go @@ -0,0 +1,275 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package batch + +import ( + "context" + "database/sql" + "fmt" + "reflect" + "sort" + "strings" + "time" + + "github.com/jmoiron/sqlx/reflectx" + + "github.com/ory/x/dbal" + + "github.com/gobuffalo/pop/v6" + "github.com/gofrs/uuid" + "github.com/pkg/errors" + + "github.com/ory/x/otelx" + "github.com/ory/x/sqlcon" + + "github.com/ory/x/sqlxx" +) + +type ( + insertQueryArgs struct { + TableName string + ColumnsDecl string + Columns []string + Placeholders string + } + quoter interface { + Quote(key string) string + } + TracerConnection struct { + Tracer *otelx.Tracer + Connection *pop.Connection + } +) + +func buildInsertQueryArgs[T any](ctx context.Context, dialect string, mapper *reflectx.Mapper, quoter quoter, models []*T) insertQueryArgs { + var ( + v T + model = pop.NewModel(v, ctx) + + columns []string + quotedColumns []string + placeholders []string + placeholderRow []string + ) + + for _, col := range model.Columns().Cols { + columns = append(columns, col.Name) + placeholderRow = append(placeholderRow, "?") + } + + // We sort for the sole reason that the test snapshots are deterministic. + sort.Strings(columns) + + for _, col := range columns { + quotedColumns = append(quotedColumns, quoter.Quote(col)) + } + + // We generate a list (for every row one) of VALUE statements here that + // will be substituted by their column values later: + // + // (?, ?, ?, ?), + // (?, ?, ?, ?), + // (?, ?, ?, ?) + for _, m := range models { + m := reflect.ValueOf(m) + + pl := make([]string, len(placeholderRow)) + copy(pl, placeholderRow) + + // There is a special case - when using CockroachDB we want to generate + // UUIDs using "gen_random_uuid()" which ends up in a VALUE statement of: + // + // (gen_random_uuid(), ?, ?, ?), + for k := range placeholderRow { + if columns[k] != "id" { + continue + } + + field := mapper.FieldByName(m, columns[k]) + val, ok := field.Interface().(uuid.UUID) + if !ok { + continue + } + + if val == uuid.Nil && dialect == dbal.DriverCockroachDB { + pl[k] = "gen_random_uuid()" + break + } + } + + placeholders = append(placeholders, fmt.Sprintf("(%s)", strings.Join(pl, ", "))) + } + + return insertQueryArgs{ + TableName: quoter.Quote(model.TableName()), + ColumnsDecl: strings.Join(quotedColumns, ", "), + Columns: columns, + Placeholders: strings.Join(placeholders, ",\n"), + } +} + +func buildInsertQueryValues[T any](dialect string, mapper *reflectx.Mapper, columns []string, models []*T, nowFunc func() time.Time) (values []any, err error) { + for _, m := range models { + m := reflect.ValueOf(m) + + now := nowFunc() + // Append model fields to args + for _, c := range columns { + field := mapper.FieldByName(m, c) + + switch c { + case "created_at": + if pop.IsZeroOfUnderlyingType(field.Interface()) { + field.Set(reflect.ValueOf(now)) + } + case "updated_at": + field.Set(reflect.ValueOf(now)) + case "id": + if field.Interface().(uuid.UUID) != uuid.Nil { + break // breaks switch, not for + } else if dialect == dbal.DriverCockroachDB { + // This is a special case: + // 1. We're using cockroach + // 2. It's the primary key field ("ID") + // 3. A UUID was not yet set. + // + // If all these conditions meet, the VALUE statement will look as such: + // + // (gen_random_uuid(), ?, ?, ?, ...) + // + // For that reason, we do not add the ID value to the list of arguments, + // because one of the arguments is using a built-in and thus doesn't need a value. + continue // break switch, not for + } + + id, err := uuid.NewV4() + if err != nil { + return nil, err + } + field.Set(reflect.ValueOf(id)) + } + + values = append(values, field.Interface()) + + // Special-handling for *sqlxx.NullTime: mapper.FieldByName sets this to a zero time.Time, + // but we want a nil pointer instead. + if i, ok := field.Interface().(*sqlxx.NullTime); ok { + if time.Time(*i).IsZero() { + field.Set(reflect.Zero(field.Type())) + } + } + } + } + + return values, nil +} + +// Create batch-inserts the given models into the database using a single INSERT statement. +// The models are either all created or none. +func Create[T any](ctx context.Context, p *TracerConnection, models []*T) (err error) { + ctx, span := p.Tracer.Tracer().Start(ctx, "persistence.sql.batch.Create") + defer otelx.End(span, &err) + + if len(models) == 0 { + return nil + } + + var v T + model := pop.NewModel(v, ctx) + + conn := p.Connection + quoter, ok := conn.Dialect.(quoter) + if !ok { + return errors.Errorf("store is not a quoter: %T", conn.Store) + } + + queryArgs := buildInsertQueryArgs(ctx, conn.Dialect.Name(), conn.TX.Mapper, quoter, models) + values, err := buildInsertQueryValues(conn.Dialect.Name(), conn.TX.Mapper, queryArgs.Columns, models, func() time.Time { return time.Now().UTC().Truncate(time.Microsecond) }) + if err != nil { + return err + } + + var returningClause string + if conn.Dialect.Name() != dbal.DriverMySQL { + // PostgreSQL, CockroachDB, SQLite support RETURNING. + returningClause = fmt.Sprintf("RETURNING %s", model.IDField()) + } + + query := conn.Dialect.TranslateSQL(fmt.Sprintf( + "INSERT INTO %s (%s) VALUES\n%s\n%s", + queryArgs.TableName, + queryArgs.ColumnsDecl, + queryArgs.Placeholders, + returningClause, + )) + + rows, err := conn.TX.QueryContext(ctx, query, values...) + if err != nil { + return sqlcon.HandleError(err) + } + defer rows.Close() + + // Hydrate the models from the RETURNING clause. + // + // Databases not supporting RETURNING will just return 0 rows. + count := 0 + for rows.Next() { + if err := rows.Err(); err != nil { + return sqlcon.HandleError(err) + } + + if err := setModelID(rows, pop.NewModel(models[count], ctx)); err != nil { + return err + } + count++ + } + + if err := rows.Err(); err != nil { + return sqlcon.HandleError(err) + } + + if err := rows.Close(); err != nil { + return sqlcon.HandleError(err) + } + + return sqlcon.HandleError(err) +} + +// setModelID was copy & pasted from pop. It basically sets +// the primary key to the given value read from the SQL row. +func setModelID(row *sql.Rows, model *pop.Model) error { + el := reflect.ValueOf(model.Value).Elem() + fbn := el.FieldByName("ID") + if !fbn.IsValid() { + return errors.New("model does not have a field named id") + } + + pkt, err := model.PrimaryKeyType() + if err != nil { + return errors.WithStack(err) + } + + switch pkt { + case "UUID": + var id uuid.UUID + if err := row.Scan(&id); err != nil { + return errors.WithStack(err) + } + fbn.Set(reflect.ValueOf(id)) + default: + var id interface{} + if err := row.Scan(&id); err != nil { + return errors.WithStack(err) + } + v := reflect.ValueOf(id) + switch fbn.Kind() { + case reflect.Int, reflect.Int64: + fbn.SetInt(v.Int()) + default: + fbn.Set(reflect.ValueOf(id)) + } + } + + return nil +} diff --git a/sqlxx/batch/create_test.go b/sqlxx/batch/create_test.go new file mode 100644 index 00000000..49c0ac46 --- /dev/null +++ b/sqlxx/batch/create_test.go @@ -0,0 +1,122 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package batch + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/ory/x/dbal" + + "github.com/gofrs/uuid" + "github.com/jmoiron/sqlx/reflectx" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ory/x/snapshotx" + "github.com/ory/x/sqlxx" +) + +type ( + testModel struct { + ID uuid.UUID `db:"id"` + NID uuid.UUID `db:"nid"` + String string `db:"string"` + Int int `db:"int"` + NullTimePtr *sqlxx.NullTime `db:"null_time_ptr"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` + } + testQuoter struct{} +) + +func (i testModel) TableName(ctx context.Context) string { + return "test_models" +} + +func (tq testQuoter) Quote(s string) string { return fmt.Sprintf("%q", s) } + +func makeModels[T any]() []*T { + models := make([]*T, 10) + for k := range models { + models[k] = new(T) + } + return models +} + +func Test_buildInsertQueryArgs(t *testing.T) { + ctx := context.Background() + t.Run("case=testModel", func(t *testing.T) { + models := makeModels[testModel]() + mapper := reflectx.NewMapper("db") + args := buildInsertQueryArgs(ctx, "other", mapper, testQuoter{}, models) + snapshotx.SnapshotT(t, args) + + query := fmt.Sprintf("INSERT INTO %s (%s) VALUES\n%s", args.TableName, args.ColumnsDecl, args.Placeholders) + assert.Equal(t, `INSERT INTO "test_models" ("created_at", "id", "int", "nid", "null_time_ptr", "string", "updated_at") VALUES +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?), +(?, ?, ?, ?, ?, ?, ?)`, query) + }) + + t.Run("case=cockroach", func(t *testing.T) { + models := makeModels[testModel]() + for k := range models { + if k%3 == 0 { + models[k].ID = uuid.FromStringOrNil(fmt.Sprintf("ae0125a9-2786-4ada-82d2-d169cf75047%d", k)) + } + } + mapper := reflectx.NewMapper("db") + args := buildInsertQueryArgs(ctx, "cockroach", mapper, testQuoter{}, models) + snapshotx.SnapshotT(t, args) + }) +} + +func Test_buildInsertQueryValues(t *testing.T) { + t.Run("case=testModel", func(t *testing.T) { + model := &testModel{ + String: "string", + Int: 42, + } + mapper := reflectx.NewMapper("db") + + nowFunc := func() time.Time { + return time.Time{} + } + t.Run("case=cockroach", func(t *testing.T) { + values, err := buildInsertQueryValues(dbal.DriverCockroachDB, mapper, []string{"created_at", "updated_at", "id", "string", "int", "null_time_ptr", "traits"}, []*testModel{model}, nowFunc) + require.NoError(t, err) + snapshotx.SnapshotT(t, values) + }) + + t.Run("case=others", func(t *testing.T) { + values, err := buildInsertQueryValues("other", mapper, []string{"created_at", "updated_at", "id", "string", "int", "null_time_ptr", "traits"}, []*testModel{model}, nowFunc) + require.NoError(t, err) + + assert.NotNil(t, model.CreatedAt) + assert.Equal(t, model.CreatedAt, values[0]) + + assert.NotNil(t, model.UpdatedAt) + assert.Equal(t, model.UpdatedAt, values[1]) + + assert.NotZero(t, model.ID) + assert.Equal(t, model.ID, values[2]) + + assert.Equal(t, model.String, values[3]) + assert.Equal(t, model.Int, values[4]) + + assert.Nil(t, model.NullTimePtr) + + }) + }) +} From 33c73f37fa66d4631109a2c4ff400754c154ddbc Mon Sep 17 00:00:00 2001 From: Lars Lehtonen Date: Fri, 10 Nov 2023 01:11:57 -0800 Subject: [PATCH 39/56] fix: decoderx dropped errors (#741) --- decoderx/http.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/decoderx/http.go b/decoderx/http.go index 1debd2ec..2c3e1c45 100644 --- a/decoderx/http.go +++ b/decoderx/http.go @@ -409,7 +409,8 @@ func (t *HTTP) decodeURLValues(values url.Values, paths []jsonschemax.Path, o *h raw, err = sjson.SetBytes(raw, path.Name, values[key]) case []float64: for k, v := range values[key] { - if f, err := strconv.ParseFloat(v, 64); err != nil { + var f float64 + if f, err = strconv.ParseFloat(v, 64); err != nil { switch o.handleParseErrors { case ParseErrorIgnoreConversionErrors: raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), v) @@ -428,12 +429,13 @@ func (t *HTTP) decodeURLValues(values url.Values, paths []jsonschemax.Path, o *h } case []bool: for k, v := range values[key] { - if f, err := strconv.ParseBool(v); err != nil { + var b bool + if b, err = strconv.ParseBool(v); err != nil { switch o.handleParseErrors { case ParseErrorIgnoreConversionErrors: raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), v) case ParseErrorUseEmptyValueOnConversionErrors: - raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), f) + raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), b) case ParseErrorReturnOnConversionErrors: return nil, errors.WithStack(herodot.ErrBadRequest.WithReasonf("Expected value to be a boolean."). WithDetail("parse_error", err.Error()). @@ -442,7 +444,7 @@ func (t *HTTP) decodeURLValues(values url.Values, paths []jsonschemax.Path, o *h WithDetail("value", v)) } } else { - raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), f) + raw, err = sjson.SetBytes(raw, path.Name+"."+strconv.Itoa(k), b) } } case []interface{}: @@ -456,12 +458,13 @@ func (t *HTTP) decodeURLValues(values url.Values, paths []jsonschemax.Path, o *h v = "false" } - if f, err := strconv.ParseBool(v); err != nil { + var b bool + if b, err = strconv.ParseBool(v); err != nil { switch o.handleParseErrors { case ParseErrorIgnoreConversionErrors: raw, err = sjson.SetBytes(raw, path.Name, v) case ParseErrorUseEmptyValueOnConversionErrors: - raw, err = sjson.SetBytes(raw, path.Name, f) + raw, err = sjson.SetBytes(raw, path.Name, b) case ParseErrorReturnOnConversionErrors: return nil, errors.WithStack(herodot.ErrBadRequest.WithReasonf("Expected value to be a boolean."). WithDetail("parse_error", err.Error()). @@ -469,7 +472,7 @@ func (t *HTTP) decodeURLValues(values url.Values, paths []jsonschemax.Path, o *h WithDetail("value", values.Get(key))) } } else { - raw, err = sjson.SetBytes(raw, path.Name, f) + raw, err = sjson.SetBytes(raw, path.Name, b) } case float64: v := values.Get(key) @@ -480,7 +483,8 @@ func (t *HTTP) decodeURLValues(values url.Values, paths []jsonschemax.Path, o *h v = "0.0" } - if f, err := strconv.ParseFloat(v, 64); err != nil { + var f float64 + if f, err = strconv.ParseFloat(v, 64); err != nil { switch o.handleParseErrors { case ParseErrorIgnoreConversionErrors: raw, err = sjson.SetBytes(raw, path.Name, v) From e1d7bd3cb462d841458541c7b2fecc070b373030 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Fri, 10 Nov 2023 11:36:14 +0100 Subject: [PATCH 40/56] feat: improve batch create API (#745) --- sqlxx/batch/create.go | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/sqlxx/batch/create.go b/sqlxx/batch/create.go index 801dcbdd..3b106eb5 100644 --- a/sqlxx/batch/create.go +++ b/sqlxx/batch/create.go @@ -126,7 +126,9 @@ func buildInsertQueryValues[T any](dialect string, mapper *reflectx.Mapper, colu case "updated_at": field.Set(reflect.ValueOf(now)) case "id": - if field.Interface().(uuid.UUID) != uuid.Nil { + if value, ok := field.Interface().(uuid.UUID); ok && value != uuid.Nil { + break // breaks switch, not for + } else if value, ok := field.Interface().(string); ok && len(value) > 0 { break // breaks switch, not for } else if dialect == dbal.DriverCockroachDB { // This is a special case: @@ -165,9 +167,21 @@ func buildInsertQueryValues[T any](dialect string, mapper *reflectx.Mapper, colu return values, nil } +type createOptions struct { + onConflict string +} + +type option func(*createOptions) + +func OnConflictDoNothing() func(*createOptions) { + return func(o *createOptions) { + o.onConflict = "ON CONFLICT DO NOTHING" + } +} + // Create batch-inserts the given models into the database using a single INSERT statement. // The models are either all created or none. -func Create[T any](ctx context.Context, p *TracerConnection, models []*T) (err error) { +func Create[T any](ctx context.Context, p *TracerConnection, models []*T, opts ...option) (err error) { ctx, span := p.Tracer.Tracer().Start(ctx, "persistence.sql.batch.Create") defer otelx.End(span, &err) @@ -175,6 +189,11 @@ func Create[T any](ctx context.Context, p *TracerConnection, models []*T) (err e return nil } + options := &createOptions{} + for _, opt := range opts { + opt(options) + } + var v T model := pop.NewModel(v, ctx) @@ -197,10 +216,11 @@ func Create[T any](ctx context.Context, p *TracerConnection, models []*T) (err e } query := conn.Dialect.TranslateSQL(fmt.Sprintf( - "INSERT INTO %s (%s) VALUES\n%s\n%s", + "INSERT INTO %s (%s) VALUES\n%s\n%s\n%s", queryArgs.TableName, queryArgs.ColumnsDecl, queryArgs.Placeholders, + options.onConflict, returningClause, )) From 9909a38dbe0b66db7a7d3a41ac141f71372cb550 Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Wed, 11 Oct 2023 18:57:52 +0200 Subject: [PATCH 41/56] feat: add WithMaxHTTPMaxBytes option to fetcher to limit HTTP response body size --- fetcher/fetcher.go | 35 ++++++++++++++++++++++++++++------- fetcher/fetcher_test.go | 19 +++++++++++++++---- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index b58f0f51..1d6181ed 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -22,22 +22,32 @@ import ( // Fetcher is able to load file contents from http, https, file, and base64 locations. type Fetcher struct { - hc *retryablehttp.Client + hc *retryablehttp.Client + limit int64 } type opts struct { - hc *retryablehttp.Client + hc *retryablehttp.Client + limit int64 } var ErrUnknownScheme = stderrors.New("unknown scheme") // WithClient sets the http.Client the fetcher uses. -func WithClient(hc *retryablehttp.Client) func(*opts) { +func WithClient(hc *retryablehttp.Client) Modifier { return func(o *opts) { o.hc = hc } } +// WithMaxHTTPMaxBytes reads at most limit bytes from the HTTP response body, +// returning bytes.ErrToLarge if the limit would be exceeded. +func WithMaxHTTPMaxBytes(limit int64) Modifier { + return func(o *opts) { + o.limit = limit + } +} + func newOpts() *opts { return &opts{ hc: httpx.NewResilientClient(), @@ -52,7 +62,7 @@ func NewFetcher(opts ...Modifier) *Fetcher { for _, f := range opts { f(o) } - return &Fetcher{hc: o.hc} + return &Fetcher{hc: o.hc, limit: o.limit} } // Fetch fetches the file contents from the source. @@ -94,7 +104,18 @@ func (f *Fetcher) fetchRemote(ctx context.Context, source string) (*bytes.Buffer return nil, errors.Errorf("expected http response status code 200 but got %d when fetching: %s", res.StatusCode, source) } - return f.decode(res.Body) + if f.limit > 0 { + var buf bytes.Buffer + n, err := io.Copy(&buf, io.LimitReader(res.Body, f.limit+1)) + if n > f.limit { + return nil, bytes.ErrTooLarge + } + if err != nil { + return nil, err + } + return &buf, nil + } + return f.toBuffer(res.Body) } func (f *Fetcher) fetchFile(source string) (*bytes.Buffer, error) { @@ -106,10 +127,10 @@ func (f *Fetcher) fetchFile(source string) (*bytes.Buffer, error) { _ = fp.Close() }() - return f.decode(fp) + return f.toBuffer(fp) } -func (f *Fetcher) decode(r io.Reader) (*bytes.Buffer, error) { +func (f *Fetcher) toBuffer(r io.Reader) (*bytes.Buffer, error) { var b bytes.Buffer if _, err := io.Copy(&b, r); err != nil { return nil, err diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index 28c04154..32f24fab 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -4,6 +4,7 @@ package fetcher import ( + "bytes" "context" "encoding/base64" "fmt" @@ -16,7 +17,6 @@ import ( "github.com/gobuffalo/httptest" "github.com/julienschmidt/httprouter" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -67,9 +67,8 @@ func TestFetcher(t *testing.T) { t.Run("case=returns proper error on unknown scheme", func(t *testing.T) { _, err := NewFetcher().Fetch("unknown-scheme://foo") - require.NotNil(t, err) - assert.True(t, errors.Is(err, ErrUnknownScheme)) + assert.ErrorIs(t, err, ErrUnknownScheme) assert.Contains(t, err.Error(), "unknown-scheme") }) @@ -77,8 +76,20 @@ func TestFetcher(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() _, err := NewFetcher().FetchContext(ctx, "https://config.invalid") - require.NotNil(t, err) assert.ErrorIs(t, err, context.DeadlineExceeded) }) + + t.Run("case=with-limit", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(bytes.Repeat([]byte("test"), 1000)) + })) + t.Cleanup(srv.Close) + + _, err := NewFetcher(WithMaxHTTPMaxBytes(3999)).Fetch(srv.URL) + assert.ErrorIs(t, err, bytes.ErrTooLarge) + + _, err = NewFetcher(WithMaxHTTPMaxBytes(4000)).Fetch(srv.URL) + assert.NoError(t, err) + }) } From b054e33b645eb305c2b408565582b56ab88b3cd0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:51:50 +0100 Subject: [PATCH 42/56] chore(deps): bump google.golang.org/grpc from 1.53.0 to 1.56.3 (#738) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.53.0 to 1.56.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.53.0...v1.56.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 4c6617e8..91018eda 100644 --- a/go.mod +++ b/go.mod @@ -95,8 +95,8 @@ require ( golang.org/x/net v0.10.0 golang.org/x/sync v0.1.0 gonum.org/v1/plot v0.12.0 - google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.28.1 + google.golang.org/grpc v1.56.3 + google.golang.org/protobuf v1.30.0 ) require ( @@ -142,8 +142,8 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect - github.com/golang/glog v1.0.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/glog v1.1.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20221010195024-131d412537ea // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.0 // indirect @@ -210,7 +210,7 @@ require ( golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index c60eaa8c..714c8592 100644 --- a/go.sum +++ b/go.sum @@ -346,8 +346,9 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -376,8 +377,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -1414,8 +1416,8 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1437,8 +1439,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99 h1:qA8rMbz1wQ4DOFfM2ouD29DG9aHWBm6ZOy9BGxiUMmY= google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1454,8 +1456,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 5ad9c46b211d742433217e634cbb11e1104917f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 09:51:59 +0100 Subject: [PATCH 43/56] chore(deps): bump golang.org/x/image from 0.5.0 to 0.10.0 (#743) Bumps [golang.org/x/image](https://github.com/golang/image) from 0.5.0 to 0.10.0. - [Commits](https://github.com/golang/image/compare/v0.5.0...v0.10.0) --- updated-dependencies: - dependency-name: golang.org/x/image dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 91018eda..073d6084 100644 --- a/go.mod +++ b/go.mod @@ -204,9 +204,9 @@ require ( go.mongodb.org/mongo-driver v1.10.3 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect go.opentelemetry.io/otel/metric v0.33.0 // indirect - golang.org/x/image v0.5.0 // indirect + golang.org/x/image v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index 714c8592..f15881e4 100644 --- a/go.sum +++ b/go.sum @@ -1056,8 +1056,8 @@ golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1265,8 +1265,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 022d9bdce3fe41eed3b83fa8eafb6b724e5d17c3 Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Mon, 13 Nov 2023 18:50:04 +0100 Subject: [PATCH 44/56] feat: improved SSRF protection --- fetcher/fetcher_test.go | 7 +- go.mod | 75 ++++++------- go.sum | 162 ++++++++++++++++------------- httpx/private_ip_validator.go | 99 ++++-------------- httpx/private_ip_validator_test.go | 120 +++++++-------------- httpx/resilient_client.go | 54 ++++++---- httpx/resilient_client_test.go | 88 +++------------- httpx/ssrf.go | 101 ++++++++++++++++++ osx/file_test.go | 8 +- otelx/otel.go | 2 +- 10 files changed, 347 insertions(+), 369 deletions(-) create mode 100644 httpx/ssrf.go diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index 32f24fab..b23ecf5c 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/ory/x/httpx" + "github.com/hashicorp/go-retryablehttp" "github.com/gobuffalo/httptest" "github.com/julienschmidt/httprouter" @@ -35,9 +35,10 @@ func TestFetcher(t *testing.T) { _, err = file.WriteString(`{"foo":"baz"}`) require.NoError(t, err) require.NoError(t, file.Close()) - + rClient := retryablehttp.NewClient() + rClient.HTTPClient = ts.Client() for fc, fetcher := range []*Fetcher{ - NewFetcher(WithClient(httpx.NewResilientClient(httpx.ResilientClientWithClient(ts.Client())))), + NewFetcher(WithClient(rClient)), NewFetcher(), } { for k, tc := range []struct { diff --git a/go.mod b/go.mod index 073d6084..bb29e597 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module github.com/ory/x go 1.20 require ( + code.dny.dev/ssrf v0.2.0 github.com/auth0/go-jwt-middleware v1.0.1 github.com/avast/retry-go/v4 v4.3.0 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 - github.com/cenkalti/backoff/v4 v4.2.0 + github.com/cenkalti/backoff/v4 v4.2.1 github.com/cockroachdb/cockroach-go/v2 v2.2.16 github.com/dgraph-io/ristretto v0.1.1 github.com/docker/docker v20.10.24+incompatible @@ -76,27 +77,28 @@ require ( github.com/tidwall/pretty v1.2.1 github.com/tidwall/sjson v1.2.5 github.com/urfave/negroni v1.0.0 - go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.36.4 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4 - go.opentelemetry.io/contrib/propagators/b3 v1.11.1 - go.opentelemetry.io/contrib/propagators/jaeger v1.11.1 - go.opentelemetry.io/contrib/samplers/jaegerremote v0.5.2 - go.opentelemetry.io/otel v1.11.1 - go.opentelemetry.io/otel/exporters/jaeger v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.9.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.9.0 - go.opentelemetry.io/otel/exporters/zipkin v1.11.1 - go.opentelemetry.io/otel/sdk v1.11.1 - go.opentelemetry.io/otel/trace v1.11.1 - go.opentelemetry.io/proto/otlp v0.18.0 + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 + go.opentelemetry.io/contrib/propagators/b3 v1.20.0 + go.opentelemetry.io/contrib/propagators/jaeger v1.20.0 + go.opentelemetry.io/contrib/samplers/jaegerremote v0.14.0 + go.opentelemetry.io/otel v1.19.0 + go.opentelemetry.io/otel/exporters/jaeger v1.17.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 + go.opentelemetry.io/otel/exporters/zipkin v1.19.0 + go.opentelemetry.io/otel/sdk v1.19.0 + go.opentelemetry.io/otel/trace v1.19.0 + go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/goleak v1.2.1 - golang.org/x/crypto v0.9.0 - golang.org/x/mod v0.8.0 - golang.org/x/net v0.10.0 - golang.org/x/sync v0.1.0 + golang.org/x/crypto v0.15.0 + golang.org/x/mod v0.14.0 + golang.org/x/net v0.18.0 + golang.org/x/oauth2 v0.14.0 + golang.org/x/sync v0.5.0 gonum.org/v1/plot v0.12.0 - google.golang.org/grpc v1.56.3 - google.golang.org/protobuf v1.30.0 + google.golang.org/grpc v1.59.0 + google.golang.org/protobuf v1.31.0 ) require ( @@ -120,10 +122,10 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-fonts/liberation v0.2.0 // indirect github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/errors v0.20.3 // indirect github.com/go-openapi/strfmt v0.21.3 // indirect @@ -142,12 +144,12 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect - github.com/golang/glog v1.1.0 // indirect + github.com/golang/glog v1.1.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20221010195024-131d412537ea // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.12.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.13 // indirect @@ -182,11 +184,11 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.5 // indirect - github.com/openzipkin/zipkin-go v0.4.1 // indirect + github.com/openzipkin/zipkin-go v0.4.2 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/segmentio/backo-go v1.0.1 // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect @@ -202,15 +204,18 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect go.mongodb.org/mongo-driver v1.10.3 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect - go.opentelemetry.io/otel/metric v0.33.0 // indirect - golang.org/x/image v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.11.0 // indirect - golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.6.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/image v0.14.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.4.0 // indirect + golang.org/x/tools v0.15.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index f15881e4..b0dce348 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +code.dny.dev/ssrf v0.2.0 h1:wCBP990rQQ1CYfRpW+YK1+8xhwUjv189AQ3WMo1jQaI= +code.dny.dev/ssrf v0.2.0/go.mod h1:B+91l25OnyaLIeCx0WRJN5qfJ/4/ZTZxRXgm0lj/2w8= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.sr.ht/~sbinet/gg v0.3.1 h1:LNhjNn8DerC8f9DHLz6lS0YYul/b602DUxDgGkd/Aik= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= @@ -91,8 +93,8 @@ github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -184,8 +186,8 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= @@ -223,8 +225,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= @@ -346,9 +348,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -394,7 +395,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-jsonnet v0.19.0 h1:G7uJZhi8t1eg5NZ+PZJ3bU0GZ4suYGGy79BCtEswlbM= github.com/google/go-jsonnet v0.19.0/go.mod h1:5JVT33JVCoehdTj5Z2KJq1eIdt3Nb8PCmZ+W5D8U350= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -421,7 +422,7 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -445,9 +446,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.12.0 h1:kr3j8iIMR4ywO/O0rvksXaJvauGGCMg2zAZIiNZ9uIQ= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.12.0/go.mod h1:ummNFgdgLhhX7aIiy35vVmQNS0rWXknfPE0qe6fmFXg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -476,6 +476,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -599,8 +600,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -708,18 +709,17 @@ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nyaruka/phonenumbers v1.0.73/go.mod h1:3aiS+PS3DuYwkbK3xdcmRwMiPNECZ0oENH8qUT1lY7Q= github.com/nyaruka/phonenumbers v1.1.1 h1:fyoZmpLN2VCmAnc51XcrNOUVP2wT1ZzQl348ggIaXII= github.com/nyaruka/phonenumbers v1.1.1/go.mod h1:cGaEsOrLjIL0iKGqJR5Rfywy86dSkbApEpXuM9KySNA= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= @@ -729,8 +729,8 @@ github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openzipkin/zipkin-go v0.4.1 h1:kNd/ST2yLLWhaWrkgchya40TJabe8Hioj9udfPcEO5A= -github.com/openzipkin/zipkin-go v0.4.1/go.mod h1:qY0VqDSN1pOBN94dBc6w2GJlWLiovAyg7Qt6/I9HecM= +github.com/openzipkin/zipkin-go v0.4.2 h1:zjqfqHjUpPmB3c1GlCvvgsM1G4LkvqQbBDueDOCg/jA= +github.com/openzipkin/zipkin-go v0.4.2/go.mod h1:ZeVkFjuuBiSy13y8vpSDCjMi9GoI3hPpCJSBx/EYFhY= github.com/ory/analytics-go/v5 v5.0.1 h1:LX8T5B9FN8KZXOtxgN+R3I4THRRVB6+28IKgKBpXmAM= github.com/ory/analytics-go/v5 v5.0.1/go.mod h1:lWCiCjAaJkKfgR/BN5DCLMol8BjKS1x+4jxBxff/FF0= github.com/ory/dockertest/v3 v3.9.1 h1:v4dkG+dlu76goxMiTT2j8zV7s4oPPEppKT8K8p2f1kY= @@ -812,8 +812,9 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -966,37 +967,35 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.36.4 h1:toN8e0U4RWQL4f8H+1eFtaeWe/IkSM3+81qJEDOgShs= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.36.4/go.mod h1:u4OeI4ujQmFbpZOOysLUfYrRWOmEVmvzkM2zExVorXM= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4 h1:aUEBEdCa6iamGzg6fuYxDA8ThxvOG240mAvWDU+XLio= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4/go.mod h1:l2MdsbKTocpPS5nQZscqTR9jd8u96VYZdcpF8Sye7mA= -go.opentelemetry.io/contrib/propagators/b3 v1.11.1 h1:icQ6ttRV+r/2fnU46BIo/g/mPu6Rs5Ug8Rtohe3KqzI= -go.opentelemetry.io/contrib/propagators/b3 v1.11.1/go.mod h1:ECIveyMXgnl4gorxFcA7RYjJY/Ql9n20ubhbfDc3QfA= -go.opentelemetry.io/contrib/propagators/jaeger v1.11.1 h1:Gw+P9NQzw4bjNGZXsoDhwwDWLnk4Y1waF8MQZAq/eYM= -go.opentelemetry.io/contrib/propagators/jaeger v1.11.1/go.mod h1:dP/N3ZFADH8azBcZfGXEFNBXpEmPTXYcNj9rkw1+2Oc= -go.opentelemetry.io/contrib/samplers/jaegerremote v0.5.2 h1:Izp9RqrioK/y7J/RXy2c7zd83iKQ4N3td3AMNKNzHiI= -go.opentelemetry.io/contrib/samplers/jaegerremote v0.5.2/go.mod h1:Z0aRlRERn9v/3J2K+ATa6ffKyb8/i+/My/gTzFr3dII= -go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4= -go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE= -go.opentelemetry.io/otel/exporters/jaeger v1.11.1 h1:F9Io8lqWdGyIbY3/SOGki34LX/l+7OL0gXNxjqwcbuQ= -go.opentelemetry.io/otel/exporters/jaeger v1.11.1/go.mod h1:lRa2w3bQ4R4QN6zYsDgy7tEezgoKEu7Ow2g35Y75+KI= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 h1:X2GndnMCsUPh6CiY2a+frAbNsXaPLbB0soHRYhAZ5Ig= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1/go.mod h1:i8vjiSzbiUC7wOQplijSXMYUpNM93DtlS5CbUT+C6oQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.9.0 h1:NN90Cuna0CnBg8YNu1Q0V35i2E8LDByFOwHRCq/ZP9I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.9.0/go.mod h1:0EsCXjZAiiZGnLdEUXM9YjCKuuLZMYyglh2QDXcYKVA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.9.0 h1:FAF9l8Wjxi9Ad2k/vLTfHZyzXYX72C62wBGpV3G6AIo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.9.0/go.mod h1:smUdtylgc0YQiUr2PuifS4hBXhAS5xtR6WQhxP1wiNA= -go.opentelemetry.io/otel/exporters/zipkin v1.11.1 h1:JlJ3/oQoyqlrPDCfsSVFcHgGeHvZq+hr1VPWtiYCXTo= -go.opentelemetry.io/otel/exporters/zipkin v1.11.1/go.mod h1:T4S6aVwIS1+MHA+dJHCcPROtZe6ORwnv5vMKPRapsFw= -go.opentelemetry.io/otel/metric v0.33.0 h1:xQAyl7uGEYvrLAiV/09iTJlp1pZnQ9Wl793qbVvED1E= -go.opentelemetry.io/otel/metric v0.33.0/go.mod h1:QlTYc+EnYNq/M2mNk1qDDMRLpqCOj2f/r5c7Fd5FYaI= -go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZpKxs= -go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys= -go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ= -go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0 h1:2ea0IkZBsWH+HA2GkD+7+hRw2u97jzdFyRtXuO14a1s= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0/go.mod h1:4m3RnBBb+7dB9d21y510oO1pdB1V4J6smNf14WXcBFQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= +go.opentelemetry.io/contrib/propagators/b3 v1.20.0 h1:Yty9Vs4F3D6/liF1o6FNt0PvN85h/BJJ6DQKJ3nrcM0= +go.opentelemetry.io/contrib/propagators/b3 v1.20.0/go.mod h1:On4VgbkqYL18kbJlWsa18+cMNe6rYpBnPi1ARI/BrsU= +go.opentelemetry.io/contrib/propagators/jaeger v1.20.0 h1:iVhNKkMIpzyZqxk8jkDU2n4DFTD+FbpGacvooxEvyyc= +go.opentelemetry.io/contrib/propagators/jaeger v1.20.0/go.mod h1:cpSABr0cm/AH/HhbJjn+AudBVUMgZWdfN3Gb+ZqxSZc= +go.opentelemetry.io/contrib/samplers/jaegerremote v0.14.0 h1:Xg9iU9DF9V9zC6NI8sJthYqHlSWsWAQMTXM8QIErKlc= +go.opentelemetry.io/contrib/samplers/jaegerremote v0.14.0/go.mod h1:ExRuq62/gYluX5fzTTZif5WujyG51ail4APTbBUu+S4= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= +go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/exporters/zipkin v1.19.0 h1:EGY0h5mGliP9o/nIkVuLI0vRiQqmsYOcbwCuotksO1o= +go.opentelemetry.io/otel/exporters/zipkin v1.19.0/go.mod h1:JQgTGJP11yi3o4GHzIWYodhPisxANdqxF1eHwDSnJrI= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.18.0 h1:W5hyXNComRa23tGpKwG+FRAc4rfF6ZUg1JReK+QHS80= -go.opentelemetry.io/proto/otlp v0.18.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1034,8 +1033,9 @@ golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1046,7 +1046,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1056,8 +1057,8 @@ golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= -golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1081,8 +1082,9 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1133,8 +1135,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1145,8 +1148,9 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= +golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1161,8 +1165,9 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1247,8 +1252,9 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1264,15 +1270,16 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= +golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1338,16 +1345,17 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/plot v0.12.0 h1:y1ZNmfz/xHuHvtgFe8USZVyykQo5ERXPnspQNVK15Og= gonum.org/v1/plot v0.12.0/go.mod h1:PgiMf9+3A3PnZdJIciIXmyN1FwdAA6rXELSN761oQkw= @@ -1377,6 +1385,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1416,9 +1426,12 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1440,8 +1453,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99 h1:qA8rMbz1wQ4DOFfM2ouD29DG9aHWBm6ZOy9BGxiUMmY= google.golang.org/grpc/examples v0.0.0-20210304020650-930c79186c99/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1457,8 +1470,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1466,6 +1479,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/mold.v2 v2.2.0/go.mod h1:XMyyRsGtakkDPbxXbrA5VODo6bUXyvoDjLd5l3T0XoA= diff --git a/httpx/private_ip_validator.go b/httpx/private_ip_validator.go index 09e20732..f644d4c4 100644 --- a/httpx/private_ip_validator.go +++ b/httpx/private_ip_validator.go @@ -6,12 +6,10 @@ package httpx import ( "fmt" "net" - "net/http" + "net/netip" "net/url" - "syscall" - "time" - "github.com/gobwas/glob" + "code.dny.dev/ssrf" "github.com/pkg/errors" ) @@ -69,83 +67,28 @@ func DisallowIPPrivateAddresses(ipOrHostnameOrURL string) error { } for _, ip := range ips { - if ip.IsPrivate() || ip.IsLoopback() || ip.IsUnspecified() { - return ErrPrivateIPAddressDisallowed(fmt.Errorf("%s is not a public IP address", ip)) - } - } - - return nil -} - -var _ http.RoundTripper = (*NoInternalIPRoundTripper)(nil) - -// NoInternalIPRoundTripper is a RoundTripper that disallows internal IP addresses. -type NoInternalIPRoundTripper struct { - http.RoundTripper - internalIPExceptions []string -} - -func (n NoInternalIPRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { - rt := http.DefaultTransport - if n.RoundTripper != nil { - rt = n.RoundTripper - } - - incoming := IncomingRequestURL(request) - incoming.RawQuery = "" - incoming.RawFragment = "" - for _, exception := range n.internalIPExceptions { - compiled, err := glob.Compile(exception, '.', '/') + ip, err := netip.ParseAddr(ip.String()) if err != nil { - return nil, err + return ErrPrivateIPAddressDisallowed(errors.WithStack(err)) // should be unreacheable } - if compiled.Match(incoming.String()) { - return rt.RoundTrip(request) - } - } - - if err := DisallowIPPrivateAddresses(incoming.Hostname()); err != nil { - return nil, err - } - - return rt.RoundTrip(request) -} -var NoInternalDialer = &net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - Control: func(network, address string, _ syscall.RawConn) error { - if !(network == "tcp4" || network == "tcp6") { - return ErrPrivateIPAddressDisallowed(fmt.Errorf("%s is not a safe network type", network)) - } - - host, _, err := net.SplitHostPort(address) - if err != nil { - return ErrPrivateIPAddressDisallowed(fmt.Errorf("%s is not a valid host/port pair: %s", address, err)) - } - - ip := net.ParseIP(host) - if ip == nil { - return ErrPrivateIPAddressDisallowed(fmt.Errorf("%s is not a valid IP address", host)) - } - - if ip.IsPrivate() || ip.IsLoopback() || ip.IsUnspecified() { - return ErrPrivateIPAddressDisallowed(fmt.Errorf("%s is not a public IP address", ip)) + if ip.Is4() { + for _, deny := range ssrf.IPv4DeniedPrefixes { + if deny.Contains(ip) { + return ErrPrivateIPAddressDisallowed(fmt.Errorf("%s is not a public IP address", ip)) + } + } + } else { + if !ssrf.IPv6GlobalUnicast.Contains(ip) { + return ErrPrivateIPAddressDisallowed(fmt.Errorf("%s is not a public IP address", ip)) + } + for _, net := range ssrf.IPv6DeniedPrefixes { + if net.Contains(ip) { + return ErrPrivateIPAddressDisallowed(fmt.Errorf("%s is not a public IP address", ip)) + } + } } + } - return nil - }, -} - -// NoInternalTransport -// -// DEPRECATED: do not use -var NoInternalTransport http.RoundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: NoInternalDialer.DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, + return nil } diff --git a/httpx/private_ip_validator_test.go b/httpx/private_ip_validator_test.go index 055a58ff..e3520ffc 100644 --- a/httpx/private_ip_validator_test.go +++ b/httpx/private_ip_validator_test.go @@ -4,13 +4,10 @@ package httpx import ( - "net" "net/http" - "net/url" "testing" "github.com/pkg/errors" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -29,9 +26,12 @@ func TestIsAssociatedIPAllowed(t *testing.T) { "0.0.0.0", "10.255.255.255", "::1", + "100::1", + "fe80::1", + "169.254.169.254", // AWS instance metadata service } { t.Run("case="+disallowed, func(t *testing.T) { - require.Error(t, DisallowIPPrivateAddresses(disallowed)) + assert.Error(t, DisallowIPPrivateAddresses(disallowed)) }) } } @@ -50,104 +50,58 @@ func (n noOpRoundTripper) RoundTrip(request *http.Request) (*http.Response, erro var _ http.RoundTripper = new(noOpRoundTripper) -type errRoundTripper struct{} +type errRoundTripper struct{ err error } -var fakeErr = errors.New("error") +var errNotOnWhitelist = errors.New("OK") +var errOnWhitelist = errors.New("OK (on whitelist)") func (n errRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { - return nil, fakeErr + return nil, n.err } var _ http.RoundTripper = new(errRoundTripper) +// TestInternalRespectsRoundTripper tests if the RoundTripper picks the correct +// underlying transport for two allowed requests. func TestInternalRespectsRoundTripper(t *testing.T) { - rt := &NoInternalIPRoundTripper{RoundTripper: &errRoundTripper{}, internalIPExceptions: []string{ - "https://127.0.0.1/foo", - }} + rt := &noInternalIPRoundTripper{ + onWhitelist: &errRoundTripper{errOnWhitelist}, + notOnWhitelist: &errRoundTripper{errNotOnWhitelist}, + internalIPExceptions: []string{ + "https://127.0.0.1/foo", + }} req, err := http.NewRequest("GET", "https://google.com/foo", nil) require.NoError(t, err) _, err = rt.RoundTrip(req) - require.ErrorIs(t, err, fakeErr) + require.ErrorIs(t, err, errNotOnWhitelist) req, err = http.NewRequest("GET", "https://127.0.0.1/foo", nil) require.NoError(t, err) _, err = rt.RoundTrip(req) - require.ErrorIs(t, err, fakeErr) + require.ErrorIs(t, err, errOnWhitelist) } func TestAllowExceptions(t *testing.T) { - rt := &NoInternalIPRoundTripper{internalIPExceptions: []string{"http://localhost/asdf"}} - - _, err := rt.RoundTrip(&http.Request{ - Host: "localhost", - URL: &url.URL{Scheme: "http", Path: "/asdf", Host: "localhost"}, - Header: http.Header{ - "Host": []string{"localhost"}, - }, - }) - // assert that the error is eiher nil or a dial error. - if err != nil { - opErr := new(net.OpError) - require.ErrorAs(t, err, &opErr) - require.Equal(t, "dial", opErr.Op) - } - - _, err = rt.RoundTrip(&http.Request{ - Host: "localhost", - URL: &url.URL{Scheme: "http", Path: "/not-asdf", Host: "localhost"}, - Header: http.Header{ - "Host": []string{"localhost"}, - }, - }) - require.Error(t, err) -} + rt := noInternalIPRoundTripper{ + onWhitelist: &errRoundTripper{errOnWhitelist}, + notOnWhitelist: &errRoundTripper{errNotOnWhitelist}, + internalIPExceptions: []string{ + "http://localhost/asdf", + }} + + req, err := http.NewRequest("GET", "http://localhost/asdf", nil) + require.NoError(t, err) + _, err = rt.RoundTrip(req) + require.ErrorIs(t, err, errOnWhitelist) -func assertErrorContains(msg string) assert.ErrorAssertionFunc { - return func(t assert.TestingT, err error, i ...interface{}) bool { - if !assert.Error(t, err, i...) { - return false - } - return assert.Contains(t, err.Error(), msg) - } -} + req, err = http.NewRequest("GET", "http://localhost/not-asdf", nil) + require.NoError(t, err) + _, err = rt.RoundTrip(req) + require.ErrorIs(t, err, errNotOnWhitelist) -func TestNoInternalDialer(t *testing.T) { - for _, tt := range []struct { - name string - network string - address string - assertErr assert.ErrorAssertionFunc - }{{ - name: "TCP public is allowed", - network: "tcp", - address: "www.google.de:443", - assertErr: assert.NoError, - }, { - name: "TCP private is denied", - network: "tcp", - address: "localhost:443", - assertErr: assertErrorContains("is not a public IP address"), - }, { - name: "UDP public is denied", - network: "udp", - address: "www.google.de:443", - assertErr: assertErrorContains("not a safe network type"), - }, { - name: "UDP public is denied", - network: "udp", - address: "www.google.de:443", - assertErr: assertErrorContains("not a safe network type"), - }, { - name: "UNIX sockets are denied", - network: "unix", - address: "/etc/passwd", - assertErr: assertErrorContains("not a safe network type"), - }} { - - t.Run("case="+tt.name, func(t *testing.T) { - _, err := NoInternalDialer.Dial(tt.network, tt.address) - tt.assertErr(t, err) - }) - } + req, err = http.NewRequest("GET", "http://127.0.0.1", nil) + require.NoError(t, err) + _, err = rt.RoundTrip(req) + require.ErrorIs(t, err, errNotOnWhitelist) } diff --git a/httpx/resilient_client.go b/httpx/resilient_client.go index 4ab77276..360379fd 100644 --- a/httpx/resilient_client.go +++ b/httpx/resilient_client.go @@ -4,6 +4,7 @@ package httpx import ( + "context" "io" "log" "net/http" @@ -11,6 +12,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/trace" + "golang.org/x/oauth2" "github.com/hashicorp/go-retryablehttp" @@ -19,6 +21,8 @@ import ( type resilientOptions struct { c *http.Client + oauthConfig *oauth2.Config + oauthToken *oauth2.Token l interface{} retryWaitMin time.Duration retryWaitMax time.Duration @@ -31,7 +35,7 @@ type resilientOptions struct { func newResilientOptions() *resilientOptions { connTimeout := time.Minute return &resilientOptions{ - c: &http.Client{Timeout: connTimeout}, + c: &http.Client{Timeout: connTimeout, Transport: http.DefaultTransport}, retryWaitMin: 1 * time.Second, retryWaitMax: 30 * time.Second, retryMax: 4, @@ -42,13 +46,6 @@ func newResilientOptions() *resilientOptions { // ResilientOptions is a set of options for the ResilientClient. type ResilientOptions func(o *resilientOptions) -// ResilientClientWithClient sets the underlying http client to use. -func ResilientClientWithClient(c *http.Client) ResilientOptions { - return func(o *resilientOptions) { - o.c = c - } -} - // ResilientClientWithTracer wraps the http clients transport with a tracing instrumentation func ResilientClientWithTracer(tracer trace.Tracer) ResilientOptions { return func(o *resilientOptions) { @@ -114,23 +111,38 @@ func NewResilientClient(opts ...ResilientOptions) *retryablehttp.Client { } if o.noInternalIPs { - o.c.Transport = &NoInternalIPRoundTripper{ - RoundTripper: o.c.Transport, - internalIPExceptions: o.internalIPExceptions, - } + o.c.Transport = NewNoInternalIPRoundTripper(o.internalIPExceptions) } if o.tracer != nil { o.c.Transport = otelhttp.NewTransport(o.c.Transport) } - return &retryablehttp.Client{ - HTTPClient: o.c, - Logger: o.l, - RetryWaitMin: o.retryWaitMin, - RetryWaitMax: o.retryWaitMax, - RetryMax: o.retryMax, - CheckRetry: retryablehttp.DefaultRetryPolicy, - Backoff: retryablehttp.DefaultBackoff, - } + cl := retryablehttp.NewClient() + cl.HTTPClient = o.c + cl.Logger = o.l + cl.RetryWaitMin = o.retryWaitMin + cl.RetryWaitMax = o.retryWaitMax + cl.RetryMax = o.retryMax + cl.CheckRetry = retryablehttp.DefaultRetryPolicy + cl.Backoff = retryablehttp.DefaultBackoff + return cl +} + +// SetOAuth2 modifies the given client to enable OAuth2 authentication. Requests +// with the client should always use the returned context. +// +// client := http.NewResilientClient(opts...) +// ctx, client = httpx.SetOAuth2(ctx, client, oauth2Config, oauth2Token) +// req, err := retryablehttp.NewRequestWithContext(ctx, ...) +// if err != nil { /* ... */ } +// res, err := client.Do(req) +func SetOAuth2(ctx context.Context, cl *retryablehttp.Client, c OAuth2Config, t *oauth2.Token) (context.Context, *retryablehttp.Client) { + ctx = context.WithValue(ctx, oauth2.HTTPClient, cl.HTTPClient) + cl.HTTPClient = c.Client(ctx, t) + return ctx, cl +} + +type OAuth2Config interface { + Client(context.Context, *oauth2.Token) *http.Client } diff --git a/httpx/resilient_client_test.go b/httpx/resilient_client_test.go index 8dc4fc8d..917dfc85 100644 --- a/httpx/resilient_client_test.go +++ b/httpx/resilient_client_test.go @@ -10,8 +10,6 @@ import ( "net/url" "testing" - "go.opentelemetry.io/otel" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -37,75 +35,23 @@ func TestNoPrivateIPs(t *testing.T) { ResilientClientAllowInternalIPRequestsTo(allowedURL, allowedGlob), ) - for destination, passes := range map[string]bool{ - "http://127.0.0.1:" + port: false, - "http://localhost:" + port: false, - "http://192.168.178.5:" + port: false, - allowedURL: true, - "http://localhost:" + port + "/glob/bar": true, - "http://localhost:" + port + "/glob/bar/baz": false, - "http://localhost:" + port + "/FOOBAR": false, - } { - _, err := c.Get(destination) - if !passes { - require.Error(t, err) - assert.Contains(t, err.Error(), "is not a public IP address") - } else { - require.NoError(t, err) + for i := 0; i < 10; i++ { + for destination, passes := range map[string]bool{ + "http://127.0.0.1:" + port: false, + "http://localhost:" + port: false, + "http://192.168.178.5:" + port: false, + allowedURL: true, + "http://localhost:" + port + "/glob/bar": true, + "http://localhost:" + port + "/glob/bar/baz": false, + "http://localhost:" + port + "/FOOBAR": false, + } { + _, err := c.Get(destination) + if !passes { + require.Errorf(t, err, "dest = %s", destination) + assert.Containsf(t, err.Error(), "is not a permitted destination", "dest = %s", destination) + } else { + require.NoErrorf(t, err, "dest = %s", destination) + } } } } - -var errClient = &http.Client{Transport: errRoundTripper{}} - -func TestNoPrivateIPsRespectsWrappedClient(t *testing.T) { - c := NewResilientClient( - ResilientClientWithMaxRetry(1), - ResilientClientDisallowInternalIPs(), - ResilientClientWithClient(errClient), - ) - _, err := c.Get("https://google.com") - require.ErrorIs(t, err, fakeErr) -} - -func TestClientWithTracerRespectsWrappedClient(t *testing.T) { - tracer := otel.Tracer("github.com/ory/x/httpx test") - c := NewResilientClient( - ResilientClientWithMaxRetry(1), - ResilientClientWithTracer(tracer), - ResilientClientWithClient(errClient), - ) - _, err := c.Get("https://google.com") - require.ErrorIs(t, err, fakeErr) -} - -func TestClientWithMultiConfigRespectsWrapperClient(t *testing.T) { - tracer := otel.Tracer("github.com/ory/x/httpx test") - c := NewResilientClient( - ResilientClientWithMaxRetry(1), - ResilientClientWithTracer(tracer), - ResilientClientDisallowInternalIPs(), - ResilientClientWithClient(errClient), - ) - _, err := c.Get("https://google.com") - require.ErrorIs(t, err, fakeErr) -} - -func TestClientWithTracer(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte("Hello, world!")) - })) - t.Cleanup(ts.Close) - - tracer := otel.Tracer("github.com/ory/x/httpx test") - c := NewResilientClient( - ResilientClientWithTracer(tracer), - ) - - target, err := url.ParseRequestURI(ts.URL) - require.NoError(t, err) - - _, err = c.Get(target.String()) - - assert.NoError(t, err) -} diff --git a/httpx/ssrf.go b/httpx/ssrf.go new file mode 100644 index 00000000..6d81cc72 --- /dev/null +++ b/httpx/ssrf.go @@ -0,0 +1,101 @@ +// Copyright © 2023 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package httpx + +import ( + "net" + "net/http" + "net/netip" + "time" + + "code.dny.dev/ssrf" + "github.com/gobwas/glob" +) + +var _ http.RoundTripper = (*noInternalIPRoundTripper)(nil) + +type noInternalIPRoundTripper struct { + onWhitelist, notOnWhitelist http.RoundTripper + internalIPExceptions []string +} + +// NewNoInternalIPRoundTripper creates a RoundTripper that disallows +// non-publicly routable IP addresses, except for URLs matching the given +// exception globs. +func NewNoInternalIPRoundTripper(exceptions []string) http.RoundTripper { + if len(exceptions) > 0 { + prohibitInternal := newSSRFTransport(ssrf.New( + ssrf.WithAnyPort(), + ssrf.WithNetworks("tcp4", "tcp6"), + )) + + allowInternal := newSSRFTransport(ssrf.New( + ssrf.WithAnyPort(), + ssrf.WithNetworks("tcp4", "tcp6"), + ssrf.WithAllowedV4Prefixes( + netip.MustParsePrefix("10.0.0.0/8"), // Private-Use (RFC 1918) + netip.MustParsePrefix("127.0.0.0/8"), // Loopback (RFC 1122, Section 3.2.1.3)) + netip.MustParsePrefix("169.254.0.0/16"), // Link Local (RFC 3927) + netip.MustParsePrefix("172.16.0.0/12"), // Private-Use (RFC 1918) + netip.MustParsePrefix("192.168.0.0/16"), // Private-Use (RFC 1918) + ), + ssrf.WithAllowedV6Prefixes( + netip.MustParsePrefix("::1/128"), // Loopback (RFC 4193) + netip.MustParsePrefix("fc00::/7"), // Unique Local (RFC 4193) + ), + )) + return noInternalIPRoundTripper{ + onWhitelist: allowInternal, + notOnWhitelist: prohibitInternal, + internalIPExceptions: exceptions, + } + } + prohibitInternal := newSSRFTransport(ssrf.New( + ssrf.WithAnyPort(), + ssrf.WithNetworks("tcp4", "tcp6"), + )) + return noInternalIPRoundTripper{ + onWhitelist: prohibitInternal, + notOnWhitelist: prohibitInternal, + } +} + +// RoundTrip implements http.RoundTripper. +func (n noInternalIPRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + incoming := IncomingRequestURL(request) + incoming.RawQuery = "" + incoming.RawFragment = "" + for _, exception := range n.internalIPExceptions { + compiled, err := glob.Compile(exception, '.', '/') + if err != nil { + return nil, err + } + if compiled.Match(incoming.String()) { + return n.onWhitelist.RoundTrip(request) + } + } + + return n.notOnWhitelist.RoundTrip(request) +} + +func newSSRFTransport(g *ssrf.Guardian) http.RoundTripper { + t := newDefaultTransport() + t.DialContext = (&net.Dialer{Control: g.Safe}).DialContext + return t +} + +func newDefaultTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } +} diff --git a/osx/file_test.go b/osx/file_test.go index ef1de190..d7ff173f 100644 --- a/osx/file_test.go +++ b/osx/file_test.go @@ -10,10 +10,9 @@ import ( "net/http/httptest" "testing" + "github.com/hashicorp/go-retryablehttp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/ory/x/httpx" ) var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { @@ -27,6 +26,9 @@ func TestReadFileFromAllSources(t *testing.T) { sslTS := httptest.NewTLSServer(handler) defer sslTS.Close() + rClient := retryablehttp.NewClient() + rClient.HTTPClient = sslTS.Client() + for k, tc := range []struct { opts []Option src string @@ -49,7 +51,7 @@ func TestReadFileFromAllSources(t *testing.T) { {src: ts.URL, expectedBody: "hello world"}, {src: sslTS.URL, expectedErrContains: "x509:"}, - {src: sslTS.URL, expectedBody: "hello world", opts: []Option{WithHTTPClient(httpx.NewResilientClient(httpx.ResilientClientWithClient(sslTS.Client())))}}, + {src: sslTS.URL, expectedBody: "hello world", opts: []Option{WithHTTPClient(rClient)}}, {src: sslTS.URL, expectedErr: "http(s) loader disabled", opts: []Option{WithDisabledHTTPLoader()}}, {src: "file://stub/text.txt", expectedErr: "file loader disabled", opts: []Option{WithDisabledFileLoader()}}, diff --git a/otelx/otel.go b/otelx/otel.go index 25e7fb44..6bf7111e 100644 --- a/otelx/otel.go +++ b/otelx/otel.go @@ -89,7 +89,7 @@ func (t *Tracer) WithOTLP(other trace.Tracer) *Tracer { return &Tracer{other} } -// Provider returns a TracerProvider which in turn yieds this tracer unmodified. +// Provider returns a TracerProvider which in turn yields this tracer unmodified. func (t *Tracer) Provider() trace.TracerProvider { return tracerProvider{t.Tracer()} } From 5616e978fe364a7bf34ff525a74d5703a127fe77 Mon Sep 17 00:00:00 2001 From: Romain Caire Date: Wed, 29 Nov 2023 07:24:21 +0100 Subject: [PATCH 45/56] feat: Add Vowels & NoVowels to the AlphaUpper charsets (#749) --- randx/sequence.go | 4 ++++ randx/sequence_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/randx/sequence.go b/randx/sequence.go index f3fc5450..0b0163ac 100644 --- a/randx/sequence.go +++ b/randx/sequence.go @@ -21,6 +21,10 @@ var ( AlphaUpperNum = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") // AlphaLower contains runes [abcdefghijklmnopqrstuvwxyz]. AlphaLower = []rune("abcdefghijklmnopqrstuvwxyz") + // AlphaUpperVowels contains runes [AEIOUY]. + AlphaUpperVowels = []rune("AEIOUY") + // AlphaUpperNoVowels contains runes [BCDFGHJKLMNPQRSTVWXZ]. + AlphaUpperNoVowels = []rune("BCDFGHJKLMNPQRSTVWXZ") // AlphaUpper contains runes [ABCDEFGHIJKLMNOPQRSTUVWXYZ]. AlphaUpper = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ") // Numeric contains runes [0123456789]. diff --git a/randx/sequence_test.go b/randx/sequence_test.go index c4d4bf97..7131fa01 100644 --- a/randx/sequence_test.go +++ b/randx/sequence_test.go @@ -18,6 +18,8 @@ func TestRunePatterns(t *testing.T) { {Alpha, "[a-zA-Z]{52}"}, {AlphaLower, "[a-z]{26}"}, {AlphaUpper, "[A-Z]{26}"}, + {AlphaUpperVowels, "[AEIOUY]{6}"}, + {AlphaUpperNoVowels, "[^AEIOUY]{20}"}, {AlphaNum, "[a-zA-Z0-9]{62}"}, {AlphaLowerNum, "[a-z0-9]{36}"}, {AlphaUpperNum, "[A-Z0-9]{36}"}, @@ -38,6 +40,8 @@ func TestRuneSequenceMatchesPattern(t *testing.T) { {Alpha, "[a-zA-Z]+", 25}, {AlphaLower, "[a-z]+", 46}, {AlphaUpper, "[A-Z]+", 21}, + {AlphaUpperVowels, "[AEIOUY]+", 12}, + {AlphaUpperNoVowels, "[^AEIOUY]+", 42}, {AlphaNum, "[a-zA-Z0-9]+", 123}, {AlphaLowerNum, "[a-z0-9]+", 41}, {AlphaUpperNum, "[A-Z0-9]+", 94914}, From bb9fbed9f6b40f17a715af7a2c6f0ceee55cbe95 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Dec 2023 10:16:31 +0000 Subject: [PATCH 46/56] chore(deps): bump github.com/go-jose/go-jose/v3 from 3.0.0 to 3.0.1 Bumps [github.com/go-jose/go-jose/v3](https://github.com/go-jose/go-jose) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/go-jose/go-jose/releases) - [Changelog](https://github.com/go-jose/go-jose/blob/v3/CHANGELOG.md) - [Commits](https://github.com/go-jose/go-jose/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: github.com/go-jose/go-jose/v3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bb29e597..d41b979e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/ghodss/yaml v1.0.0 github.com/go-bindata/go-bindata v3.1.2+incompatible - github.com/go-jose/go-jose/v3 v3.0.0 + github.com/go-jose/go-jose/v3 v3.0.1 github.com/go-openapi/jsonpointer v0.19.5 github.com/go-openapi/runtime v0.24.2 github.com/go-sql-driver/mysql v1.7.0 diff --git a/go.sum b/go.sum index b0dce348..5e03fd94 100644 --- a/go.sum +++ b/go.sum @@ -212,8 +212,8 @@ github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmn github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= -github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= +github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= From 0910275296562d479eeb9a189100b9634ab9e632 Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Thu, 28 Dec 2023 12:18:21 +0100 Subject: [PATCH 47/56] feat: update otel (#751) --- go.mod | 20 ++++++++++---------- go.sum | 40 ++++++++++++++++++++-------------------- otelx/otel.go | 6 +++++- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index d41b979e..2e368351 100644 --- a/go.mod +++ b/go.mod @@ -77,18 +77,18 @@ require ( github.com/tidwall/pretty v1.2.1 github.com/tidwall/sjson v1.2.5 github.com/urfave/negroni v1.0.0 - go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/contrib/propagators/b3 v1.20.0 go.opentelemetry.io/contrib/propagators/jaeger v1.20.0 - go.opentelemetry.io/contrib/samplers/jaegerremote v0.14.0 - go.opentelemetry.io/otel v1.19.0 + go.opentelemetry.io/contrib/samplers/jaegerremote v0.15.1 + go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/exporters/jaeger v1.17.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 - go.opentelemetry.io/otel/exporters/zipkin v1.19.0 - go.opentelemetry.io/otel/sdk v1.19.0 - go.opentelemetry.io/otel/trace v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 + go.opentelemetry.io/otel/exporters/zipkin v1.21.0 + go.opentelemetry.io/otel/sdk v1.21.0 + go.opentelemetry.io/otel/trace v1.21.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/goleak v1.2.1 golang.org/x/crypto v0.15.0 @@ -204,7 +204,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect go.mongodb.org/mongo-driver v1.10.3 // indirect - go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/image v0.14.0 // indirect golang.org/x/sys v0.14.0 // indirect diff --git a/go.sum b/go.sum index 5e03fd94..7437bc01 100644 --- a/go.sum +++ b/go.sum @@ -967,32 +967,32 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0 h1:2ea0IkZBsWH+HA2GkD+7+hRw2u97jzdFyRtXuO14a1s= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0/go.mod h1:4m3RnBBb+7dB9d21y510oO1pdB1V4J6smNf14WXcBFQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 h1:gbhw/u49SS3gkPWiYweQNJGm/uJN5GkI/FrosxSHT7A= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/contrib/propagators/b3 v1.20.0 h1:Yty9Vs4F3D6/liF1o6FNt0PvN85h/BJJ6DQKJ3nrcM0= go.opentelemetry.io/contrib/propagators/b3 v1.20.0/go.mod h1:On4VgbkqYL18kbJlWsa18+cMNe6rYpBnPi1ARI/BrsU= go.opentelemetry.io/contrib/propagators/jaeger v1.20.0 h1:iVhNKkMIpzyZqxk8jkDU2n4DFTD+FbpGacvooxEvyyc= go.opentelemetry.io/contrib/propagators/jaeger v1.20.0/go.mod h1:cpSABr0cm/AH/HhbJjn+AudBVUMgZWdfN3Gb+ZqxSZc= -go.opentelemetry.io/contrib/samplers/jaegerremote v0.14.0 h1:Xg9iU9DF9V9zC6NI8sJthYqHlSWsWAQMTXM8QIErKlc= -go.opentelemetry.io/contrib/samplers/jaegerremote v0.14.0/go.mod h1:ExRuq62/gYluX5fzTTZif5WujyG51ail4APTbBUu+S4= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/contrib/samplers/jaegerremote v0.15.1 h1:Qb+5A+JbIjXwO7l4HkRUhgIn4Bzz0GNS2q+qdmSx+0c= +go.opentelemetry.io/contrib/samplers/jaegerremote v0.15.1/go.mod h1:G4vNCm7fRk0kjZ6pGNLo5SpLxAUvOfSrcaegnT8TPck= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= -go.opentelemetry.io/otel/exporters/zipkin v1.19.0 h1:EGY0h5mGliP9o/nIkVuLI0vRiQqmsYOcbwCuotksO1o= -go.opentelemetry.io/otel/exporters/zipkin v1.19.0/go.mod h1:JQgTGJP11yi3o4GHzIWYodhPisxANdqxF1eHwDSnJrI= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= +go.opentelemetry.io/otel/exporters/zipkin v1.21.0 h1:D+Gv6lSfrFBWmQYyxKjDd0Zuld9SRXpIrEsKZvE4DO4= +go.opentelemetry.io/otel/exporters/zipkin v1.21.0/go.mod h1:83oMKR6DzmHisFOW3I+yIMGZUTjxiWaiBI8M8+TU5zE= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= diff --git a/otelx/otel.go b/otelx/otel.go index 6bf7111e..23b9e28e 100644 --- a/otelx/otel.go +++ b/otelx/otel.go @@ -5,6 +5,7 @@ package otelx import ( "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/embedded" "github.com/ory/x/logrusx" "github.com/ory/x/stringsx" @@ -91,13 +92,16 @@ func (t *Tracer) WithOTLP(other trace.Tracer) *Tracer { // Provider returns a TracerProvider which in turn yields this tracer unmodified. func (t *Tracer) Provider() trace.TracerProvider { - return tracerProvider{t.Tracer()} + return tracerProvider{t: t.Tracer()} } type tracerProvider struct { + embedded.TracerProvider t trace.Tracer } +func (tp tracerProvider) tracerProvider() {} + var _ trace.TracerProvider = tracerProvider{} // Tracer implements trace.TracerProvider. From e77628a496c15d5da5dc178284fe5714729eaa57 Mon Sep 17 00:00:00 2001 From: aeneasr <3372410+aeneasr@users.noreply.github.com> Date: Thu, 28 Dec 2023 17:31:23 +0100 Subject: [PATCH 48/56] chore: update otel --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2e368351..8ad39416 100644 --- a/go.mod +++ b/go.mod @@ -79,8 +79,8 @@ require ( github.com/urfave/negroni v1.0.0 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 - go.opentelemetry.io/contrib/propagators/b3 v1.20.0 - go.opentelemetry.io/contrib/propagators/jaeger v1.20.0 + go.opentelemetry.io/contrib/propagators/b3 v1.21.0 + go.opentelemetry.io/contrib/propagators/jaeger v1.21.1 go.opentelemetry.io/contrib/samplers/jaegerremote v0.15.1 go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/exporters/jaeger v1.17.0 diff --git a/go.sum b/go.sum index 7437bc01..52c7755a 100644 --- a/go.sum +++ b/go.sum @@ -971,10 +971,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0. go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= -go.opentelemetry.io/contrib/propagators/b3 v1.20.0 h1:Yty9Vs4F3D6/liF1o6FNt0PvN85h/BJJ6DQKJ3nrcM0= -go.opentelemetry.io/contrib/propagators/b3 v1.20.0/go.mod h1:On4VgbkqYL18kbJlWsa18+cMNe6rYpBnPi1ARI/BrsU= -go.opentelemetry.io/contrib/propagators/jaeger v1.20.0 h1:iVhNKkMIpzyZqxk8jkDU2n4DFTD+FbpGacvooxEvyyc= -go.opentelemetry.io/contrib/propagators/jaeger v1.20.0/go.mod h1:cpSABr0cm/AH/HhbJjn+AudBVUMgZWdfN3Gb+ZqxSZc= +go.opentelemetry.io/contrib/propagators/b3 v1.21.0 h1:uGdgDPNzwQWRwCXJgw/7h29JaRqcq9B87Iv4hJDKAZw= +go.opentelemetry.io/contrib/propagators/b3 v1.21.0/go.mod h1:D9GQXvVGT2pzyTfp1QBOnD1rzKEWzKjjwu5q2mslCUI= +go.opentelemetry.io/contrib/propagators/jaeger v1.21.1 h1:f4beMGDKiVzg9IcX7/VuWVy+oGdjx3dNJ72YehmtY5k= +go.opentelemetry.io/contrib/propagators/jaeger v1.21.1/go.mod h1:U9jhkEl8d1LL+QXY7q3kneJWJugiN3kZJV2OWz3hkBY= go.opentelemetry.io/contrib/samplers/jaegerremote v0.15.1 h1:Qb+5A+JbIjXwO7l4HkRUhgIn4Bzz0GNS2q+qdmSx+0c= go.opentelemetry.io/contrib/samplers/jaegerremote v0.15.1/go.mod h1:G4vNCm7fRk0kjZ6pGNLo5SpLxAUvOfSrcaegnT8TPck= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= From 1c6366eee66a0f71494e8e680c75e1aa7d1fe4b6 Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Thu, 4 Jan 2024 11:53:50 +0100 Subject: [PATCH 49/56] fix: consider only non-empty values when parsing pagination parameters from URLs --- pagination/keysetpagination/header.go | 10 ++++++---- pagination/keysetpagination/paginator_test.go | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/pagination/keysetpagination/header.go b/pagination/keysetpagination/header.go index e523859b..93dd43d8 100644 --- a/pagination/keysetpagination/header.go +++ b/pagination/keysetpagination/header.go @@ -11,6 +11,8 @@ import ( "strings" "github.com/pkg/errors" + + "github.com/ory/x/stringsx" ) // Pagination Request Parameters @@ -94,8 +96,8 @@ func Header(w http.ResponseWriter, u *url.URL, p *Paginator) { // Parse returns the pagination options from the URL query. func Parse(q url.Values, p PageTokenConstructor) ([]Option, error) { var opts []Option - if q.Has("page_token") { - pageToken, err := url.QueryUnescape(q.Get("page_token")) + if pt := stringsx.Coalesce(q["page_token"]...); pt != "" { + pageToken, err := url.QueryUnescape(pt) if err != nil { return nil, errors.WithStack(err) } @@ -105,8 +107,8 @@ func Parse(q url.Values, p PageTokenConstructor) ([]Option, error) { } opts = append(opts, WithToken(parsed)) } - if q.Get("page_size") != "" { - size, err := strconv.Atoi(q.Get("page_size")) + if ps := stringsx.Coalesce(q["page_size"]...); ps != "" { + size, err := strconv.Atoi(ps) if err != nil { return nil, errors.WithStack(err) } diff --git a/pagination/keysetpagination/paginator_test.go b/pagination/keysetpagination/paginator_test.go index bbc5d1af..edcff86e 100644 --- a/pagination/keysetpagination/paginator_test.go +++ b/pagination/keysetpagination/paginator_test.go @@ -168,6 +168,26 @@ func TestParse(t *testing.T) { _, err := Parse(url.Values{"page_size": {"invalid-int"}}, NewStringPageToken) require.ErrorIs(t, err, strconv.ErrSyntax) }) + + t.Run("empty tokens and page sizes work as if unset, empty values are skipped", func(t *testing.T) { + opts, err := Parse(url.Values{}, NewStringPageToken) + require.NoError(t, err) + paginator := GetPaginator(append(opts, WithDefaultToken(StringPageToken("default")))...) + assert.Equal(t, "default", paginator.Token().Encode()) + assert.Equal(t, 100, paginator.Size()) + + opts, err = Parse(url.Values{"page_token": {""}, "page_size": {""}}, NewStringPageToken) + require.NoError(t, err) + paginator = GetPaginator(append(opts, WithDefaultToken(StringPageToken("default2")))...) + assert.Equal(t, "default2", paginator.Token().Encode()) + assert.Equal(t, 100, paginator.Size()) + + opts, err = Parse(url.Values{"page_token": {"", "foo", ""}, "page_size": {"", "123", ""}}, NewStringPageToken) + require.NoError(t, err) + paginator = GetPaginator(append(opts, WithDefaultToken(StringPageToken("default3")))...) + assert.Equal(t, "foo", paginator.Token().Encode()) + assert.Equal(t, 123, paginator.Size()) + }) } func TestPaginateWithAdditionalColumn(t *testing.T) { From 728f15df927bf0f76c603c952d193afd2d45c071 Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Thu, 4 Jan 2024 11:57:13 +0100 Subject: [PATCH 50/56] fix: bump licence formatter to not update all the copyright years --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 671b3b1b..1012cf75 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ $(foreach dep, $(GO_DEPENDENCIES), $(eval $(call make-go-dependency, $(dep)))) $(call make-lint-dependency) .bin/ory: Makefile - curl https://raw.githubusercontent.com/ory/meta/master/install.sh | bash -s -- -b .bin ory v0.1.48 + curl https://raw.githubusercontent.com/ory/meta/master/install.sh | bash -s -- -b .bin ory v0.2.2 touch .bin/ory .PHONY: format From 83b6be28cba5f5b4960a57305257eb40c0605eea Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Fri, 19 Jan 2024 09:13:23 +0100 Subject: [PATCH 51/56] feat: add caching option to fetcher (#753) * fix: pin CRDB version in dockertest helper * feat: add caching option to fetcher --- fetcher/fetcher.go | 82 +++++++++++++++++++++++----------- fetcher/fetcher_test.go | 41 ++++++++++++++++- go.mod | 2 +- go.sum | 22 ++------- jwksx/fetcher_v2.go | 3 +- sqlcon/dockertest/cockroach.go | 2 +- 6 files changed, 103 insertions(+), 49 deletions(-) diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 1d6181ed..11ae6c9d 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -6,13 +6,16 @@ package fetcher import ( "bytes" "context" + "crypto/sha256" "encoding/base64" stderrors "errors" "io" "net/http" "os" "strings" + "time" + "github.com/dgraph-io/ristretto" "github.com/hashicorp/go-retryablehttp" "github.com/pkg/errors" @@ -24,11 +27,15 @@ import ( type Fetcher struct { hc *retryablehttp.Client limit int64 + cache *ristretto.Cache + ttl time.Duration } type opts struct { hc *retryablehttp.Client limit int64 + cache *ristretto.Cache + ttl time.Duration } var ErrUnknownScheme = stderrors.New("unknown scheme") @@ -48,6 +55,16 @@ func WithMaxHTTPMaxBytes(limit int64) Modifier { } } +func WithCache(cache *ristretto.Cache, ttl time.Duration) Modifier { + return func(o *opts) { + if ttl < 0 { + return + } + o.cache = cache + o.ttl = ttl + } +} + func newOpts() *opts { return &opts{ hc: httpx.NewResilientClient(), @@ -62,41 +79,58 @@ func NewFetcher(opts ...Modifier) *Fetcher { for _, f := range opts { f(o) } - return &Fetcher{hc: o.hc, limit: o.limit} + return &Fetcher{hc: o.hc, limit: o.limit, cache: o.cache, ttl: o.ttl} } // Fetch fetches the file contents from the source. -func (f *Fetcher) Fetch(source string) (*bytes.Buffer, error) { +func (f *Fetcher) Fetch(source string) ([]byte, error) { return f.FetchContext(context.Background(), source) } // FetchContext fetches the file contents from the source and allows to pass a // context that is used for HTTP requests. -func (f *Fetcher) FetchContext(ctx context.Context, source string) (*bytes.Buffer, error) { +func (f *Fetcher) FetchContext(ctx context.Context, source string) ([]byte, error) { switch s := stringsx.SwitchPrefix(source); { case s.HasPrefix("http://"), s.HasPrefix("https://"): return f.fetchRemote(ctx, source) case s.HasPrefix("file://"): - return f.fetchFile(strings.Replace(source, "file://", "", 1)) + return f.fetchFile(strings.TrimPrefix(source, "file://")) case s.HasPrefix("base64://"): - src, err := base64.StdEncoding.DecodeString(strings.Replace(source, "base64://", "", 1)) + src, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(source, "base64://")) if err != nil { - return nil, errors.Wrapf(err, "rule: %s", source) + return nil, errors.Wrapf(err, "base64decode: %s", source) } - return bytes.NewBuffer(src), nil + return src, nil default: return nil, errors.Wrap(ErrUnknownScheme, s.ToUnknownPrefixErr().Error()) } } -func (f *Fetcher) fetchRemote(ctx context.Context, source string) (*bytes.Buffer, error) { +func (f *Fetcher) fetchRemote(ctx context.Context, source string) (b []byte, err error) { + if f.cache != nil { + cacheKey := sha256.Sum256([]byte(source)) + if v, ok := f.cache.Get(cacheKey[:]); ok { + cached := v.([]byte) + b = make([]byte, len(cached)) + copy(b, cached) + return b, nil + } + defer func() { + if err == nil && len(b) > 0 { + toCache := make([]byte, len(b)) + copy(toCache, b) + f.cache.SetWithTTL(cacheKey[:], toCache, int64(len(toCache)), f.ttl) + } + }() + } + req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodGet, source, nil) if err != nil { - return nil, errors.Wrapf(err, "rule: %s", source) + return nil, errors.Wrapf(err, "new request: %s", source) } res, err := f.hc.Do(req) if err != nil { - return nil, errors.Wrapf(err, "rule: %s", source) + return nil, errors.Wrap(err, source) } defer res.Body.Close() @@ -113,27 +147,23 @@ func (f *Fetcher) fetchRemote(ctx context.Context, source string) (*bytes.Buffer if err != nil { return nil, err } - return &buf, nil + return buf.Bytes(), nil } - return f.toBuffer(res.Body) + return io.ReadAll(res.Body) } -func (f *Fetcher) fetchFile(source string) (*bytes.Buffer, error) { +func (f *Fetcher) fetchFile(source string) ([]byte, error) { fp, err := os.Open(source) // #nosec:G304 if err != nil { - return nil, errors.Wrapf(err, "unable to fetch from source: %s", source) + return nil, errors.Wrapf(err, "unable to open file: %s", source) } - defer func() { - _ = fp.Close() - }() - - return f.toBuffer(fp) -} - -func (f *Fetcher) toBuffer(r io.Reader) (*bytes.Buffer, error) { - var b bytes.Buffer - if _, err := io.Copy(&b, r); err != nil { - return nil, err + defer fp.Close() + b, err := io.ReadAll(fp) + if err != nil { + return nil, errors.Wrapf(err, "unable to read file: %s", source) + } + if err := fp.Close(); err != nil { + return nil, errors.Wrapf(err, "unable to close file: %s", source) } - return &b, nil + return b, nil } diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index b23ecf5c..56c3cf7b 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -10,9 +10,11 @@ import ( "fmt" "net/http" "os" + "sync/atomic" "testing" "time" + "github.com/dgraph-io/ristretto" "github.com/hashicorp/go-retryablehttp" "github.com/gobuffalo/httptest" @@ -61,7 +63,7 @@ func TestFetcher(t *testing.T) { t.Run(fmt.Sprintf("config=%d/case=%d", fc, k), func(t *testing.T) { actual, err := fetcher.Fetch(tc.source) require.NoError(t, err) - assert.JSONEq(t, tc.expect, actual.String()) + assert.JSONEq(t, tc.expect, string(actual)) }) } } @@ -93,4 +95,41 @@ func TestFetcher(t *testing.T) { _, err = NewFetcher(WithMaxHTTPMaxBytes(4000)).Fetch(srv.URL) assert.NoError(t, err) }) + + t.Run("case=with-cache", func(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("toodaloo")) + atomic.AddInt32(&hits, 1) + })) + t.Cleanup(srv.Close) + + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 100 * 10, + MaxCost: 100, + BufferItems: 64, + }) + require.NoError(t, err) + + f := NewFetcher(WithCache(cache, time.Hour)) + + res, err := f.Fetch(srv.URL) + require.NoError(t, err) + require.Equal(t, "toodaloo", string(res)) + + require.EqualValues(t, 1, atomic.LoadInt32(&hits)) + + f.cache.Wait() + + for i := 0; i < 100; i++ { + res2, err := f.Fetch(srv.URL) + require.NoError(t, err) + require.Equal(t, "toodaloo", string(res2)) + if &res == &res2 { + t.Fatalf("cache should not return the same pointer") + } + } + + require.EqualValues(t, 1, atomic.LoadInt32(&hits)) + }) } diff --git a/go.mod b/go.mod index 8ad39416..c8f27e71 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/cenkalti/backoff/v4 v4.2.1 - github.com/cockroachdb/cockroach-go/v2 v2.2.16 + github.com/cockroachdb/cockroach-go/v2 v2.3.5 github.com/dgraph-io/ristretto v0.1.1 github.com/docker/docker v20.10.24+incompatible github.com/evanphx/json-patch/v5 v5.6.0 diff --git a/go.sum b/go.sum index 52c7755a..ca100e8b 100644 --- a/go.sum +++ b/go.sum @@ -117,8 +117,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/cockroachdb/cockroach-go/v2 v2.2.16 h1:t9dmZuC9J2W8IDQDSIGXmP+fBuEJSsrGXxWQz4cYqBY= -github.com/cockroachdb/cockroach-go/v2 v2.2.16/go.mod h1:xZ2VHjUEb/cySv0scXBx7YsBnHtLHkR1+w/w73b5i3M= +github.com/cockroachdb/cockroach-go/v2 v2.3.5 h1:Khtm8K6fTTz/ZCWPzU9Ne3aOW9VyAnj4qIPCJgKtwK0= +github.com/cockroachdb/cockroach-go/v2 v2.3.5/go.mod h1:1wNJ45eSXW9AnOc3skntW9ZUZz6gxrQK3cOj3rK+BC8= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= @@ -499,8 +499,6 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.12.0/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= -github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= @@ -518,7 +516,6 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= @@ -527,28 +524,22 @@ github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01C github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI= -github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jandelgado/gcov2lcov v1.0.4/go.mod h1:NnSxK6TMlg1oGDBfGelGbjgorT5/L3cchlbtgFYZSss= github.com/jandelgado/gcov2lcov v1.0.5 h1:rkBt40h0CVK4oCb8Dps950gvfd1rYvQ8+cWa346lVU0= github.com/jandelgado/gcov2lcov v1.0.5/go.mod h1:NnSxK6TMlg1oGDBfGelGbjgorT5/L3cchlbtgFYZSss= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -628,7 +619,6 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/luna-duclos/instrumentedsql v1.1.3 h1:t7mvC0z1jUt5A0UQ6I/0H31ryymuQRnJcWCiqV3lSAA= @@ -841,9 +831,8 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1030,7 +1019,6 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= @@ -1242,7 +1230,6 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1511,9 +1498,6 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.3.5/go.mod h1:EGCWefLFQSVFrHGy4J8EtiHCWX5Q8t0yz2Jt9aKkGzU= -gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.23.5/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/jwksx/fetcher_v2.go b/jwksx/fetcher_v2.go index a0ea7590..cba71506 100644 --- a/jwksx/fetcher_v2.go +++ b/jwksx/fetcher_v2.go @@ -4,6 +4,7 @@ package jwksx import ( + "bytes" "context" "crypto/sha256" "time" @@ -156,7 +157,7 @@ func (f *FetcherNext) fetch(ctx context.Context, location string, opts *fetcherN return nil, err } - set, err := jwk.ParseReader(result) + set, err := jwk.ParseReader(bytes.NewBuffer(result)) if err != nil { return nil, errors.WithStack(herodot.ErrBadRequest.WithReason("failed to parse JWK set").WithWrap(err)) } diff --git a/sqlcon/dockertest/cockroach.go b/sqlcon/dockertest/cockroach.go index 084bf7a6..ecd0bf2d 100644 --- a/sqlcon/dockertest/cockroach.go +++ b/sqlcon/dockertest/cockroach.go @@ -11,7 +11,7 @@ import ( ) func NewLocalTestCRDBServer(t testing.TB) string { - ts, err := testserver.NewTestServer() + ts, err := testserver.NewTestServer(testserver.CustomVersionOpt("23.1.13")) require.NoError(t, err) t.Cleanup(ts.Stop) From 135889c3da82f5d05a8749b30ab5220286c6596a Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Mon, 22 Jan 2024 19:31:54 +0100 Subject: [PATCH 52/56] feat: pooled process-isolated Jsonnet VM feat: add -0 flag to jsonnet subcommand to evaluate multiple snippets from a single process --- .github/workflows/format.yml | 2 +- .github/workflows/licenses.yml | 2 +- .github/workflows/test.yml | 5 +- Makefile | 4 +- go.mod | 6 +- go.sum | 22 ++- jsonnetsecure/cmd.go | 78 ++++++--- jsonnetsecure/jsonnet.go | 15 +- jsonnetsecure/jsonnet_pool.go | 245 +++++++++++++++++++++++++++++ jsonnetsecure/jsonnet_processvm.go | 13 +- jsonnetsecure/jsonnet_test.go | 177 ++++++++++++++++++--- jsonnetsecure/null.go | 22 +++ jsonnetsecure/provider.go | 13 +- 13 files changed, 537 insertions(+), 67 deletions(-) create mode 100644 jsonnetsecure/jsonnet_pool.go create mode 100644 jsonnetsecure/null.go diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 80515a61..b59c85d3 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: "1.20" + go-version: "1.21" - run: make format - name: Indicate formatting issues run: git diff HEAD --exit-code --color diff --git a/.github/workflows/licenses.yml b/.github/workflows/licenses.yml index cab99605..8871ccb2 100644 --- a/.github/workflows/licenses.yml +++ b/.github/workflows/licenses.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: "1.20" + go-version: "1.21" - uses: actions/setup-node@v2 with: node-version: "18" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51fcda13..87728efb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: "1.20" + go-version: "1.21" - run: | go test -tags sqlite -failfast -short -timeout=20m $(go list ./... | grep -v sqlcon | grep -v watcherx | grep -v pkgerx | grep -v configx) shell: bash @@ -55,10 +55,11 @@ jobs: uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: "1.20" + go-version: "1.21" - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: + version: v1.55.2 skip-go-installation: true args: --timeout 5m - name: Install cockroach DB diff --git a/Makefile b/Makefile index 1012cf75..f36eb156 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ licenses: .bin/licenses node_modules # checks open-source licenses GOBIN=$(shell pwd)/.bin go install golang.org/x/tools/cmd/goimports@latest .bin/golangci-lint: Makefile - bash <(curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh) -d -b .bin v1.54.2 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b .bin v1.55.2 .bin/licenses: Makefile curl https://raw.githubusercontent.com/ory/ci/master/licenses/install | sh @@ -56,7 +56,7 @@ resetdb: .PHONY: lint lint: .bin/golangci-lint - GO111MODULE=on golangci-lint run -v ./... + GO111MODULE=on .bin/golangci-lint run -v ./... .PHONY: migrations-render migrations-render: .bin/ory diff --git a/go.mod b/go.mod index c8f27e71..af69e6d9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ory/x -go 1.20 +go 1.21 require ( code.dny.dev/ssrf v0.2.0 @@ -29,13 +29,14 @@ require ( github.com/goccy/go-yaml v1.9.6 github.com/gofrs/uuid v4.3.0+incompatible github.com/golang/mock v1.6.0 - github.com/google/go-jsonnet v0.19.0 + github.com/google/go-jsonnet v0.20.0 github.com/gorilla/websocket v1.5.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/hashicorp/go-retryablehttp v0.7.1 github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf github.com/jackc/pgconn v1.13.0 github.com/jackc/pgx/v4 v4.17.2 + github.com/jackc/puddle/v2 v2.1.2 github.com/jandelgado/gcov2lcov v1.0.5 github.com/jmoiron/sqlx v1.3.5 github.com/julienschmidt/httprouter v1.3.0 @@ -205,6 +206,7 @@ require ( github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect go.mongodb.org/mongo-driver v1.10.3 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.uber.org/atomic v1.10.0 // indirect golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect golang.org/x/image v0.14.0 // indirect golang.org/x/sys v0.14.0 // indirect diff --git a/go.sum b/go.sum index ca100e8b..ee5101f2 100644 --- a/go.sum +++ b/go.sum @@ -179,7 +179,6 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= @@ -194,6 +193,7 @@ github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9 github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -396,8 +396,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-jsonnet v0.19.0 h1:G7uJZhi8t1eg5NZ+PZJ3bU0GZ4suYGGy79BCtEswlbM= -github.com/google/go-jsonnet v0.19.0/go.mod h1:5JVT33JVCoehdTj5Z2KJq1eIdt3Nb8PCmZ+W5D8U350= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-jsonnet v0.20.0 h1:WG4TTSARuV7bSm4PMB4ohjxe33IHT5WVTrJSU33uT4g= +github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -423,6 +424,7 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -456,6 +458,7 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= @@ -536,6 +539,8 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle/v2 v2.1.2 h1:0f7vaaXINONKTsxYDn4otOAiJanX/BMeAtY//BXqzlg= +github.com/jackc/puddle/v2 v2.1.2/go.mod h1:2lpufsF5mRHO6SuZkm0fNYxM6SWHfvyFj62KwNzgels= github.com/jandelgado/gcov2lcov v1.0.4/go.mod h1:NnSxK6TMlg1oGDBfGelGbjgorT5/L3cchlbtgFYZSss= github.com/jandelgado/gcov2lcov v1.0.5 h1:rkBt40h0CVK4oCb8Dps950gvfd1rYvQ8+cWa346lVU0= github.com/jandelgado/gcov2lcov v1.0.5/go.mod h1:NnSxK6TMlg1oGDBfGelGbjgorT5/L3cchlbtgFYZSss= @@ -593,6 +598,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -710,6 +716,7 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= @@ -827,7 +834,6 @@ github.com/segmentio/backo-go v1.0.1/go.mod h1:9/Rh6yILuLysoQnZ2oNooD2g7aBnvM7r/ github.com/segmentio/conf v1.2.0/go.mod h1:Y3B9O/PqqWqjyxyWWseyj/quPEtMu1zDp/kVbSWWaB0= github.com/segmentio/go-snakecase v1.1.0/go.mod h1:jk1miR5MS7Na32PZUykG89Arm+1BUSYhuGR6b7+hJto= github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZgS0fb1Ahys= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -989,6 +995,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1237,7 +1245,6 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= @@ -1344,6 +1351,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= gonum.org/v1/plot v0.12.0 h1:y1ZNmfz/xHuHvtgFe8USZVyykQo5ERXPnspQNVK15Og= gonum.org/v1/plot v0.12.0/go.mod h1:PgiMf9+3A3PnZdJIciIXmyN1FwdAA6rXELSN761oQkw= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1465,6 +1473,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1500,6 +1509,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= +gotest.tools/v3 v3.2.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1510,8 +1520,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/jsonnetsecure/cmd.go b/jsonnetsecure/cmd.go index d8112535..d1215269 100644 --- a/jsonnetsecure/cmd.go +++ b/jsonnetsecure/cmd.go @@ -4,6 +4,8 @@ package jsonnetsecure import ( + "bufio" + "fmt" "io" "github.com/pkg/errors" @@ -11,44 +13,76 @@ import ( ) func NewJsonnetCmd() *cobra.Command { - var ( - params processParameters - ) + var null bool cmd := &cobra.Command{ Use: "jsonnet", Short: "Run Jsonnet as a CLI command", Hidden: true, RunE: func(cmd *cobra.Command, args []string) error { - if err := params.DecodeFrom(cmd.InOrStdin()); err != nil { - return err + if null { + return scan(cmd.OutOrStdout(), cmd.InOrStdin()) } - vm := MakeSecureVM() - for _, it := range params.ExtCodes { - vm.ExtCode(it.Key, it.Value) - } - for _, it := range params.ExtVars { - vm.ExtVar(it.Key, it.Value) - } - for _, it := range params.TLACodes { - vm.TLACode(it.Key, it.Value) - } - for _, it := range params.TLAVars { - vm.TLAVar(it.Key, it.Value) + input, err := io.ReadAll(cmd.InOrStdin()) + if err != nil { + return errors.Wrap(err, "failed to read from stdin") } - result, err := vm.EvaluateAnonymousSnippet(params.Filename, params.Snippet) + json, err := eval(input) if err != nil { - return errors.Wrap(err, "failed to evaluate snippet") + return errors.Wrap(err, "failed to evaluate jsonnet") } - if _, err := io.WriteString(cmd.OutOrStdout(), result); err != nil { - return errors.Wrap(err, "failed to write to stdout") + if _, err := io.WriteString(cmd.OutOrStdout(), json); err != nil { + return errors.Wrap(err, "failed to write json output") } - return nil }, } + cmd.Flags().BoolVarP(&null, "null", "0", false, + `Read multiple snippets and parameters from stdin separated by null bytes. +Output will be in the same order as inputs, separated by null bytes. +Evaluation errors will also be reported to stdout, separated by null bytes. +Non-recoverable errors are written to stderr and the program will terminate with a non-zero exit code.`) return cmd } + +func scan(w io.Writer, r io.Reader) error { + scanner := bufio.NewScanner(r) + scanner.Split(splitNull) + for scanner.Scan() { + json, err := eval(scanner.Bytes()) + if err != nil { + json = fmt.Sprintf("ERROR: %s", err) + } + if _, err := fmt.Fprintf(w, "%s%c", json, 0); err != nil { + return errors.Wrap(err, "failed to write json output") + } + } + return errors.Wrap(scanner.Err(), "failed to read from stdin") +} + +func eval(input []byte) (json string, err error) { + var params processParameters + if err := params.Decode(input); err != nil { + return "", err + } + + vm := MakeSecureVM() + + for _, it := range params.ExtCodes { + vm.ExtCode(it.Key, it.Value) + } + for _, it := range params.ExtVars { + vm.ExtVar(it.Key, it.Value) + } + for _, it := range params.TLACodes { + vm.TLACode(it.Key, it.Value) + } + for _, it := range params.TLAVars { + vm.TLAVar(it.Key, it.Value) + } + + return vm.EvaluateAnonymousSnippet(params.Filename, params.Snippet) +} diff --git a/jsonnetsecure/jsonnet.go b/jsonnetsecure/jsonnet.go index 69efc9fa..0cf7ead6 100644 --- a/jsonnetsecure/jsonnet.go +++ b/jsonnetsecure/jsonnet.go @@ -42,6 +42,7 @@ type ( jsonnetBinaryPath string args []string ctx context.Context + pool *pool } Option func(o *vmOptions) @@ -55,6 +56,13 @@ func newVMOptions() *vmOptions { } } +func WithProcessPool(p Pool) Option { + return func(o *vmOptions) { + pool, _ := p.(*pool) + o.pool = pool + } +} + func WithProcessIsolatedVM(ctx context.Context) Option { return func(o *vmOptions) { o.useProcessVM = true @@ -80,9 +88,10 @@ func MakeSecureVM(opts ...Option) VM { o(options) } - if options.useProcessVM { - vm := NewProcessVM(options) - return vm + if options.pool != nil { + return NewProcessPoolVM(options) + } else if options.useProcessVM { + return NewProcessVM(options) } else { vm := jsonnet.MakeVM() vm.Importer(new(ErrorImporter)) diff --git a/jsonnetsecure/jsonnet_pool.go b/jsonnetsecure/jsonnet_pool.go new file mode 100644 index 00000000..3447bbff --- /dev/null +++ b/jsonnetsecure/jsonnet_pool.go @@ -0,0 +1,245 @@ +// Copyright © 2024 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package jsonnetsecure + +import ( + "bufio" + "context" + "encoding/json" + "io" + "math" + "os/exec" + "strings" + "time" + + "github.com/jackc/puddle/v2" + "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/ory/x/otelx" +) + +type ( + processPoolVM struct { + path string + args []string + ctx context.Context + params processParameters + pool *pool + } + Pool interface { + Close() + private() + } + pool struct { + puddle *puddle.Pool[worker] + } + worker struct { + cmd *exec.Cmd + stdin chan<- []byte + stdout <-chan string + stderr <-chan string + } + contextKeyType string +) + +var ( + ErrProcessPoolClosed = errors.New("jsonnetsecure: process pool closed") + + _ VM = (*processPoolVM)(nil) + _ Pool = (*pool)(nil) + + contextValuePath contextKeyType = "argc" + contextValueArgs contextKeyType = "argv" +) + +func NewProcessPool(size int) Pool { + size = max(1, min(size, math.MaxInt32)) + pud, err := puddle.NewPool(&puddle.Config[worker]{ + MaxSize: int32(size), + Constructor: newWorker, + Destructor: worker.destroy, + }) + if err != nil { + panic(err) // this should never happen, see implementation of puddle.NewPool + } + go func() { + for { + time.Sleep(10 * time.Second) + for _, proc := range pud.AcquireAllIdle() { + if proc.Value().cmd.ProcessState != nil { + proc.Destroy() + } else { + proc.Release() + } + } + } + }() + return &pool{pud} +} + +func (*pool) private() {} + +func (p *pool) Close() { + p.puddle.Close() +} + +func newWorker(ctx context.Context) (_ worker, err error) { + tracer := trace.SpanFromContext(ctx).TracerProvider().Tracer("") + ctx, span := tracer.Start(ctx, "jsonnetsecure.newWorker") + defer otelx.End(span, &err) + + path, _ := ctx.Value(contextValuePath).(string) + if path == "" { + return worker{}, errors.New("newWorker: missing binary path in context") + } + args, _ := ctx.Value(contextValueArgs).([]string) + cmd := exec.Command(path, append(args, "-0")...) + cmd.Env = []string{"GOMAXPROCS=1"} + stdin, err := cmd.StdinPipe() + if err != nil { + return worker{}, errors.Wrap(err, "newWorker: failed to create stdin pipe") + } + + in := make(chan []byte) + go func(c <-chan []byte) { + for input := range c { + if _, err := stdin.Write(append(input, 0)); err != nil { + stdin.Close() + return + } + } + }(in) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return worker{}, errors.Wrap(err, "newWorker: failed to create stdout pipe") + } + stderr, err := cmd.StderrPipe() + if err != nil { + return worker{}, errors.Wrap(err, "newWorker: failed to create stderr pipe") + } + if err := cmd.Start(); err != nil { + return worker{}, errors.Wrap(err, "newWorker: failed to start process") + } + + scan := func(c chan<- string, r io.Reader) { + defer close(c) + scanner := bufio.NewScanner(r) + scanner.Split(splitNull) + for scanner.Scan() { + c <- scanner.Text() + } + if err := scanner.Err(); err != nil { + c <- "ERROR: scan: " + err.Error() + } + } + out := make(chan string) + go scan(out, stdout) + errs := make(chan string) + go scan(errs, stderr) + + return worker{ + cmd: cmd, + stdin: in, + stdout: out, + stderr: errs, + }, nil +} + +func (w worker) destroy() { + close(w.stdin) + w.cmd.Process.Kill() +} + +func (w worker) eval(ctx context.Context, processParams []byte) (output string, err error) { + tracer := trace.SpanFromContext(ctx).TracerProvider().Tracer("") + ctx, span := tracer.Start(ctx, "jsonnetsecure.worker.eval") + defer otelx.End(span, &err) + + select { + case <-ctx.Done(): + return "", ctx.Err() + case w.stdin <- processParams: + break + } + + select { + case <-ctx.Done(): + return "", ctx.Err() + case output := <-w.stdout: + return output, nil + case err := <-w.stderr: + return "", errors.New(err) + } +} + +func (vm *processPoolVM) EvaluateAnonymousSnippet(filename string, snippet string) (_ string, err error) { + tracer := trace.SpanFromContext(vm.ctx).TracerProvider().Tracer("") + ctx, span := tracer.Start(vm.ctx, "jsonnetsecure.processPoolVM.EvaluateAnonymousSnippet", trace.WithAttributes(attribute.String("filename", filename))) + defer otelx.End(span, &err) + + // TODO: maybe leave the timeout to the caller? + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + params := vm.params + params.Filename = filename + params.Snippet = snippet + pp, err := json.Marshal(params) + if err != nil { + return "", errors.Wrap(err, "jsonnetsecure: marshal") + } + + ctx = context.WithValue(ctx, contextValuePath, vm.path) + ctx = context.WithValue(ctx, contextValueArgs, vm.args) + worker, err := vm.pool.puddle.Acquire(ctx) + if err != nil { + return "", errors.Wrap(err, "jsonnetsecure: acquire") + } + + result, err := worker.Value().eval(ctx, pp) + if err != nil { + worker.Destroy() + return "", errors.Wrap(err, "jsonnetsecure: eval") + } else { + worker.Release() + } + + if strings.HasPrefix(result, "ERROR: ") { + return "", errors.New("jsonnetsecure: " + result) + } + + return result, nil +} + +func NewProcessPoolVM(opts *vmOptions) VM { + ctx := opts.ctx + if ctx == nil { + ctx = context.Background() + } + return &processPoolVM{ + path: opts.jsonnetBinaryPath, + args: opts.args, + ctx: ctx, + pool: opts.pool, + } +} + +func (vm *processPoolVM) ExtCode(key string, val string) { + vm.params.ExtCodes = append(vm.params.ExtCodes, kv{key, val}) +} + +func (vm *processPoolVM) ExtVar(key string, val string) { + vm.params.ExtVars = append(vm.params.ExtVars, kv{key, val}) +} + +func (vm *processPoolVM) TLACode(key string, val string) { + vm.params.TLACodes = append(vm.params.TLACodes, kv{key, val}) +} + +func (vm *processPoolVM) TLAVar(key string, val string) { + vm.params.TLAVars = append(vm.params.TLAVars, kv{key, val}) +} diff --git a/jsonnetsecure/jsonnet_processvm.go b/jsonnetsecure/jsonnet_processvm.go index f057156f..de5567c0 100644 --- a/jsonnetsecure/jsonnet_processvm.go +++ b/jsonnetsecure/jsonnet_processvm.go @@ -13,14 +13,12 @@ import ( "strings" "time" + "github.com/cenkalti/backoff/v4" + "github.com/pkg/errors" "go.opentelemetry.io/otel/attribute" - - "github.com/ory/x/otelx" - "go.opentelemetry.io/otel/trace" - "github.com/cenkalti/backoff/v4" - "github.com/pkg/errors" + "github.com/ory/x/otelx" ) func NewProcessVM(opts *vmOptions) VM { @@ -94,6 +92,11 @@ func (p *ProcessVM) TLAVar(key string, val string) { func (pp *processParameters) EncodeTo(w io.Writer) error { return json.NewEncoder(w).Encode(pp) } + func (pp *processParameters) DecodeFrom(r io.Reader) error { return json.NewDecoder(r).Decode(pp) } + +func (pp *processParameters) Decode(d []byte) error { + return json.Unmarshal(d, pp) +} diff --git a/jsonnetsecure/jsonnet_test.go b/jsonnetsecure/jsonnet_test.go index 1904d055..869fe792 100644 --- a/jsonnetsecure/jsonnet_test.go +++ b/jsonnetsecure/jsonnet_test.go @@ -4,10 +4,16 @@ package jsonnetsecure import ( + "bufio" "context" "errors" "fmt" + "math/rand" "os/exec" + "runtime" + "strconv" + "strings" + "sync/atomic" "testing" "time" @@ -29,6 +35,11 @@ func TestSecureVM(t *testing.T) { WithProcessIsolatedVM(context.Background()), WithJsonnetBinary(testBinary), }}, + {"process pool vm", []Option{ + WithProcessIsolatedVM(context.Background()), + WithProcessPool(procPool), + WithJsonnetBinary(testBinary), + }}, } { t.Run("options="+optCase.name, func(t *testing.T) { for i, contents := range []string{ @@ -110,10 +121,10 @@ func TestSecureVM(t *testing.T) { }) }) - t.Run("case=process isolation", func(t *testing.T) { + t.Run("case=stack overflow", func(t *testing.T) { snippet := "local f(x) = if x == 0 then [] else [f(x - 1), f(x - 1)]; f(100)" ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() + t.Cleanup(cancel) vm := MakeSecureVM( WithProcessIsolatedVM(ctx), WithJsonnetBinary(testBinary), @@ -131,6 +142,20 @@ func TestSecureVM(t *testing.T) { assert.Equal(t, exitErr.ProcessState.ExitCode(), -1) }) + t.Run("case=stack overflow pool", func(t *testing.T) { + snippet := "local f(x) = if x == 0 then [] else [f(x - 1), f(x - 1)]; f(100)" + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + t.Cleanup(cancel) + vm := MakeSecureVM( + WithProcessIsolatedVM(ctx), + WithJsonnetBinary(testBinary), + WithProcessPool(procPool), + ) + result, err := vm.EvaluateAnonymousSnippet("test", snippet) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Empty(t, result) + }) + t.Run("case=importbin", func(t *testing.T) { // importbin does not exist in the current version, but is already merged on the main branch: // https://github.com/google/go-jsonnet/commit/856bd58872418eee1cede0badea5b7b462c429eb @@ -142,22 +167,44 @@ func TestSecureVM(t *testing.T) { }) } -func standardVM(t *testing.T) VM { return jsonnet.MakeVM() } -func secureVM(t *testing.T) VM { return MakeSecureVM() } +func standardVM(t *testing.T) VM { + t.Helper() + return jsonnet.MakeVM() +} + +func secureVM(t *testing.T) VM { + t.Helper() + return MakeSecureVM() +} + func processVM(t *testing.T) VM { + t.Helper() + return MakeSecureVM( + WithProcessIsolatedVM(context.Background()), + WithJsonnetBinary(JsonnetTestBinary(t))) +} + +func poolVM(t *testing.T) VM { + t.Helper() + pool := NewProcessPool(10) + t.Cleanup(pool.Close) return MakeSecureVM( WithProcessIsolatedVM(context.Background()), + WithProcessPool(pool), WithJsonnetBinary(JsonnetTestBinary(t))) } + func assertEqualVMOutput(t *testing.T, run func(factory func(t *testing.T) VM) string) { t.Helper() expectedOut := run(standardVM) secureOut := run(secureVM) processOut := run(processVM) + poolOut := run(poolVM) assert.Equal(t, expectedOut, secureOut, "secure output incorrect") assert.Equal(t, expectedOut, processOut, "process output incorrect") + assert.Equal(t, expectedOut, poolOut, "pool output incorrect") } func TestCreateMultipleProcessVMs(t *testing.T) { @@ -180,31 +227,121 @@ func TestCreateMultipleProcessVMs(t *testing.T) { require.NoError(t, wg.Wait()) } +func TestMain(m *testing.M) { + procPool = NewProcessPool(runtime.GOMAXPROCS(0)) + defer procPool.Close() + m.Run() +} + +var ( + procPool Pool + snippet = "{a:std.extVar('a')}" +) + func BenchmarkIsolatedVM(b *testing.B) { - snippet := "{a:1}" - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - vm := MakeSecureVM( - WithProcessIsolatedVM(ctx), - WithJsonnetBinary(JsonnetTestBinary(b)), - ) + binary := JsonnetTestBinary(b) - for i := 0; i < b.N; i++ { - _, err := vm.EvaluateAnonymousSnippet("test", snippet) - if err != nil { + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + vm := MakeSecureVM( + WithProcessIsolatedVM(context.Background()), + WithJsonnetBinary(binary), + ) + i := rand.Int() + vm.ExtCode("a", strconv.Itoa(i)) + res, err := vm.EvaluateAnonymousSnippet("test", snippet) require.NoError(b, err) + require.JSONEq(b, fmt.Sprintf(`{"a": %d}`, i), res) } - } + }) +} + +func BenchmarkProcessPoolVM(b *testing.B) { + binary := JsonnetTestBinary(b) + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + vm := MakeSecureVM( + WithJsonnetBinary(binary), + WithProcessPool(procPool), + ) + i := rand.Int() + vm.ExtCode("a", strconv.Itoa(i)) + res, err := vm.EvaluateAnonymousSnippet("test", snippet) + require.NoError(b, err) + require.JSONEq(b, fmt.Sprintf(`{"a": %d}`, i), res) + } + }) } func BenchmarkRegularVM(b *testing.B) { - snippet := "{a:1}" - vm := MakeSecureVM() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + vm := MakeSecureVM() + i := rand.Int() + vm.ExtCode("a", strconv.Itoa(i)) + res, err := vm.EvaluateAnonymousSnippet("test", snippet) + require.NoError(b, err) + require.JSONEq(b, fmt.Sprintf(`{"a": %d}`, i), res) + } + }) +} + +func BenchmarkReusableProcessVM(b *testing.B) { + var ( + binary = JsonnetTestBinary(b) + cmd = exec.Command(binary, "-0") + inputs = make(chan struct{}) + stderr strings.Builder + eg errgroup.Group + count int32 = 0 + ) + stdin, err := cmd.StdinPipe() + require.NoError(b, err) + stdout, err := cmd.StdoutPipe() + require.NoError(b, err) + cmd.Stderr = &stderr + require.NoError(b, cmd.Start()) + + b.Cleanup(func() { + close(inputs) + assert.NoError(b, stdin.Close()) + assert.NoError(b, eg.Wait()) + assert.NoError(b, cmd.Wait()) + assert.Empty(b, stderr.String()) + }) - for i := 0; i < b.N; i++ { - _, err := vm.EvaluateAnonymousSnippet("test", snippet) - if err != nil { + eg.Go(func() error { + scanner := bufio.NewScanner(stdout) + scanner.Split(splitNull) + for scanner.Scan() { + c := atomic.AddInt32(&count, 1) + require.JSONEq(b, fmt.Sprintf(`{"a": %d}`, c), scanner.Text()) + } + return scanner.Err() + }) + + eg.Go(func() error { + a := 1 + for range inputs { + pp := processParameters{Snippet: snippet, ExtCodes: []kv{{"a", strconv.Itoa(a)}}} + a++ + require.NoError(b, pp.EncodeTo(stdin)) + _, err := stdin.Write([]byte{0}) require.NoError(b, err) } + return nil + }) + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + inputs <- struct{}{} + } + }) + for atomic.LoadInt32(&count) != int32(b.N) { + time.Sleep(1 * time.Millisecond) } } diff --git a/jsonnetsecure/null.go b/jsonnetsecure/null.go new file mode 100644 index 00000000..42d6e921 --- /dev/null +++ b/jsonnetsecure/null.go @@ -0,0 +1,22 @@ +// Copyright © 2024 Ory Corp +// SPDX-License-Identifier: Apache-2.0 + +package jsonnetsecure + +import "bytes" + +func splitNull(data []byte, atEOF bool) (advance int, token []byte, err error) { + // Look for a null byte; if found, return the position after it, + // the data before it, and no error. + if i := bytes.IndexByte(data, 0); i >= 0 { + return i + 1, data[0:i], nil + } + + // If we're at EOF, we have a final, non-terminated word. Return it. + if atEOF && len(data) != 0 { + return len(data), data, nil + } + + // Request more data. + return 0, nil, nil +} diff --git a/jsonnetsecure/provider.go b/jsonnetsecure/provider.go index 5990e421..dbc5dbca 100644 --- a/jsonnetsecure/provider.go +++ b/jsonnetsecure/provider.go @@ -6,6 +6,7 @@ package jsonnetsecure import ( "context" "os" + "runtime" "testing" ) @@ -21,23 +22,28 @@ type ( // com/ory/x/jsonnetsecure/cmd. TestProvider struct { jsonnetBinary string + pool Pool } // DefaultProvider provides a secure VM by calling the currently // running the current binary with the provided subcommand. DefaultProvider struct { Subcommand string + Pool Pool } ) func NewTestProvider(t *testing.T) *TestProvider { - return &TestProvider{JsonnetTestBinary(t)} + pool := NewProcessPool(runtime.GOMAXPROCS(0)) + t.Cleanup(pool.Close) + return &TestProvider{JsonnetTestBinary(t), pool} } -func (t *TestProvider) JsonnetVM(ctx context.Context) (VM, error) { +func (p *TestProvider) JsonnetVM(ctx context.Context) (VM, error) { return MakeSecureVM( WithProcessIsolatedVM(ctx), - WithJsonnetBinary(t.jsonnetBinary), + WithProcessPool(p.pool), + WithJsonnetBinary(p.jsonnetBinary), ), nil } @@ -50,5 +56,6 @@ func (p *DefaultProvider) JsonnetVM(ctx context.Context) (VM, error) { WithProcessIsolatedVM(ctx), WithJsonnetBinary(self), WithProcessArgs(p.Subcommand), + WithProcessPool(p.Pool), ), nil } From ee6571a4a08911562c291b82865a4e5dce1c7bcb Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Tue, 23 Jan 2024 17:25:39 +0100 Subject: [PATCH 53/56] fix: update snapshots --- jsonschemax/.snapshots/TestListPaths-case=0.json | 8 ++++---- .../.snapshots/TestListPathsWithRecursion-case=0.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jsonschemax/.snapshots/TestListPaths-case=0.json b/jsonschemax/.snapshots/TestListPaths-case=0.json index 5e3cc642..f9ee1c46 100644 --- a/jsonschemax/.snapshots/TestListPaths-case=0.json +++ b/jsonschemax/.snapshots/TestListPaths-case=0.json @@ -2271,7 +2271,7 @@ "Type": "", "TypeHint": 1, "Format": "", - "Pattern": {}, + "Pattern": "^[0-9]+(ns|us|ms|s|m|h)$", "Enum": null, "Constant": null, "ReadOnly": false, @@ -3314,7 +3314,7 @@ "Type": "", "TypeHint": 1, "Format": "", - "Pattern": {}, + "Pattern": "^[0-9]+(ns|us|ms|s|m|h)$", "Enum": null, "Constant": null, "ReadOnly": false, @@ -3339,7 +3339,7 @@ "Type": "", "TypeHint": 1, "Format": "", - "Pattern": {}, + "Pattern": "^[0-9]+(ns|us|ms|s|m|h)$", "Enum": null, "Constant": null, "ReadOnly": false, @@ -3364,7 +3364,7 @@ "Type": "", "TypeHint": 1, "Format": "", - "Pattern": {}, + "Pattern": "^[0-9]+(ns|us|ms|s|m|h)$", "Enum": null, "Constant": null, "ReadOnly": false, diff --git a/jsonschemax/.snapshots/TestListPathsWithRecursion-case=0.json b/jsonschemax/.snapshots/TestListPathsWithRecursion-case=0.json index e430ad7d..3e460cbe 100644 --- a/jsonschemax/.snapshots/TestListPathsWithRecursion-case=0.json +++ b/jsonschemax/.snapshots/TestListPathsWithRecursion-case=0.json @@ -155,7 +155,7 @@ "Type": "", "TypeHint": 1, "Format": "email", - "Pattern": {}, + "Pattern": ".*", "Enum": null, "Constant": null, "ReadOnly": false, @@ -197,7 +197,7 @@ "Type": "", "TypeHint": 1, "Format": "email", - "Pattern": {}, + "Pattern": ".*", "Enum": null, "Constant": null, "ReadOnly": false, From 0cdec1ab8560a73379dc2a64c3c9d3e17940f66d Mon Sep 17 00:00:00 2001 From: hackerman <3372410+aeneasr@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:18:24 +0100 Subject: [PATCH 54/56] feat: add nullable bool defaulting to false (#758) --- sqlxx/types.go | 50 +++++++++++++++++++++++++++++++++++++++++++++ sqlxx/types_test.go | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/sqlxx/types.go b/sqlxx/types.go index defadb29..2d167a46 100644 --- a/sqlxx/types.go +++ b/sqlxx/types.go @@ -163,6 +163,56 @@ func (ns *NullBool) UnmarshalJSON(data []byte) error { return errors.WithStack(json.Unmarshal(data, &ns.Bool)) } +// FalsyNullBool represents a bool that may be null. +// It JSON decodes to false if null. +// +// swagger:type bool +// swagger:model falsyNullBool +type FalsyNullBool struct { + Bool bool + Valid bool // Valid is true if Bool is not NULL +} + +// Scan implements the Scanner interface. +func (ns *FalsyNullBool) Scan(value interface{}) error { + var d = sql.NullBool{} + if err := d.Scan(value); err != nil { + return err + } + + ns.Bool = d.Bool + ns.Valid = d.Valid + return nil +} + +// Value implements the driver Valuer interface. +func (ns FalsyNullBool) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return ns.Bool, nil +} + +// MarshalJSON returns m as the JSON encoding of m. +func (ns FalsyNullBool) MarshalJSON() ([]byte, error) { + if !ns.Valid { + return []byte("false"), nil + } + return json.Marshal(ns.Bool) +} + +// UnmarshalJSON sets *m to a copy of data. +func (ns *FalsyNullBool) UnmarshalJSON(data []byte) error { + if ns == nil { + return errors.New("json.RawMessage: UnmarshalJSON on nil pointer") + } + if len(data) == 0 || string(data) == "null" { + return nil + } + ns.Valid = true + return errors.WithStack(json.Unmarshal(data, &ns.Bool)) +} + // swagger:type string // swagger:model nullString type NullString string diff --git a/sqlxx/types_test.go b/sqlxx/types_test.go index 1b686672..b19afd65 100644 --- a/sqlxx/types_test.go +++ b/sqlxx/types_test.go @@ -64,6 +64,42 @@ func TestNullBoolMarshalJSON(t *testing.T) { } } +func TestNullBoolDefaultFalseMarshalJSON(t *testing.T) { + type outer struct { + Bool *FalsyNullBool `json:"null_bool,omitempty"` + } + + for k, tc := range []struct { + in *outer + expected string + }{ + {in: &outer{&FalsyNullBool{Valid: false, Bool: true}}, expected: "{\"null_bool\":false}"}, + {in: &outer{&FalsyNullBool{Valid: false, Bool: false}}, expected: "{\"null_bool\":false}"}, + {in: &outer{&FalsyNullBool{Valid: true, Bool: true}}, expected: "{\"null_bool\":true}"}, + {in: &outer{&FalsyNullBool{Valid: true, Bool: false}}, expected: "{\"null_bool\":false}"}, + {in: &outer{}, expected: "{}"}, + } { + t.Run(fmt.Sprintf("case=%d", k), func(t *testing.T) { + out, err := json.Marshal(tc.in) + require.NoError(t, err) + assert.EqualValues(t, tc.expected, string(out)) + + var actual outer + require.NoError(t, json.Unmarshal(out, &actual)) + if tc.in.Bool == nil { + assert.Nil(t, actual.Bool) + return + } else if !tc.in.Bool.Valid { + assert.False(t, actual.Bool.Bool) + return + } + + assert.EqualValues(t, tc.in.Bool.Bool, actual.Bool.Bool) + assert.EqualValues(t, tc.in.Bool.Valid, actual.Bool.Valid) + }) + } +} + func TestNullInt64MarshalJSON(t *testing.T) { type outer struct { Int64 *NullInt64 `json:"null_int,omitempty"` From a6f7cce45ad2cb27383d1c060ba81c9822dde6dd Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Fri, 2 Feb 2024 13:38:19 +0100 Subject: [PATCH 55/56] fix: revert fetcher API for backwards-compatibility (#760) --- fetcher/fetcher.go | 14 ++++++++++++-- fetcher/fetcher_test.go | 6 +++--- jwksx/fetcher_v2.go | 3 +-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 11ae6c9d..aeda9e1a 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -83,13 +83,23 @@ func NewFetcher(opts ...Modifier) *Fetcher { } // Fetch fetches the file contents from the source. -func (f *Fetcher) Fetch(source string) ([]byte, error) { +func (f *Fetcher) Fetch(source string) (*bytes.Buffer, error) { return f.FetchContext(context.Background(), source) } // FetchContext fetches the file contents from the source and allows to pass a // context that is used for HTTP requests. -func (f *Fetcher) FetchContext(ctx context.Context, source string) ([]byte, error) { +func (f *Fetcher) FetchContext(ctx context.Context, source string) (*bytes.Buffer, error) { + b, err := f.FetchBytes(ctx, source) + if err != nil { + return nil, err + } + return bytes.NewBuffer(b), nil +} + +// FetchBytes fetches the file contents from the source and allows to pass a +// context that is used for HTTP requests. +func (f *Fetcher) FetchBytes(ctx context.Context, source string) ([]byte, error) { switch s := stringsx.SwitchPrefix(source); { case s.HasPrefix("http://"), s.HasPrefix("https://"): return f.fetchRemote(ctx, source) diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index 56c3cf7b..62241bee 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -63,7 +63,7 @@ func TestFetcher(t *testing.T) { t.Run(fmt.Sprintf("config=%d/case=%d", fc, k), func(t *testing.T) { actual, err := fetcher.Fetch(tc.source) require.NoError(t, err) - assert.JSONEq(t, tc.expect, string(actual)) + assert.JSONEq(t, tc.expect, actual.String()) }) } } @@ -115,7 +115,7 @@ func TestFetcher(t *testing.T) { res, err := f.Fetch(srv.URL) require.NoError(t, err) - require.Equal(t, "toodaloo", string(res)) + require.Equal(t, "toodaloo", res.String()) require.EqualValues(t, 1, atomic.LoadInt32(&hits)) @@ -124,7 +124,7 @@ func TestFetcher(t *testing.T) { for i := 0; i < 100; i++ { res2, err := f.Fetch(srv.URL) require.NoError(t, err) - require.Equal(t, "toodaloo", string(res2)) + require.Equal(t, "toodaloo", res2.String()) if &res == &res2 { t.Fatalf("cache should not return the same pointer") } diff --git a/jwksx/fetcher_v2.go b/jwksx/fetcher_v2.go index cba71506..a0ea7590 100644 --- a/jwksx/fetcher_v2.go +++ b/jwksx/fetcher_v2.go @@ -4,7 +4,6 @@ package jwksx import ( - "bytes" "context" "crypto/sha256" "time" @@ -157,7 +156,7 @@ func (f *FetcherNext) fetch(ctx context.Context, location string, opts *fetcherN return nil, err } - set, err := jwk.ParseReader(bytes.NewBuffer(result)) + set, err := jwk.ParseReader(result) if err != nil { return nil, errors.WithStack(herodot.ErrBadRequest.WithReason("failed to parse JWK set").WithWrap(err)) } From 041443cb3c35239afcaed502ccb603f73675524d Mon Sep 17 00:00:00 2001 From: Arne Luenser Date: Mon, 5 Feb 2024 16:55:38 +0100 Subject: [PATCH 56/56] feat: add http client option to disable IPv6 * improve connection re-use * add detailed trace events (DNS, TLS, etc) --- httpx/resilient_client.go | 31 +++++++- httpx/resilient_client_test.go | 73 +++++++++++++++++++ httpx/ssrf.go | 128 +++++++++++++++++++++------------ jsonnetsecure/jsonnet_pool.go | 3 +- 4 files changed, 186 insertions(+), 49 deletions(-) diff --git a/httpx/resilient_client.go b/httpx/resilient_client.go index 360379fd..afd66199 100644 --- a/httpx/resilient_client.go +++ b/httpx/resilient_client.go @@ -8,8 +8,10 @@ import ( "io" "log" "net/http" + "net/http/httptrace" "time" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/trace" "golang.org/x/oauth2" @@ -29,17 +31,19 @@ type resilientOptions struct { retryMax int noInternalIPs bool internalIPExceptions []string + ipV6 bool tracer trace.Tracer } func newResilientOptions() *resilientOptions { connTimeout := time.Minute return &resilientOptions{ - c: &http.Client{Timeout: connTimeout, Transport: http.DefaultTransport}, + c: &http.Client{Timeout: connTimeout}, retryWaitMin: 1 * time.Second, retryWaitMax: 30 * time.Second, retryMax: 4, l: log.New(io.Discard, "", log.LstdFlags), + ipV6: true, } } @@ -103,6 +107,12 @@ func ResilientClientAllowInternalIPRequestsTo(urlGlobs ...string) ResilientOptio } } +func ResilientClientNoIPv6() ResilientOptions { + return func(o *resilientOptions) { + o.ipV6 = false + } +} + // NewResilientClient creates a new ResilientClient. func NewResilientClient(opts ...ResilientOptions) *retryablehttp.Client { o := newResilientOptions() @@ -111,11 +121,19 @@ func NewResilientClient(opts ...ResilientOptions) *retryablehttp.Client { } if o.noInternalIPs { - o.c.Transport = NewNoInternalIPRoundTripper(o.internalIPExceptions) + o.c.Transport = &noInternalIPRoundTripper{ + onWhitelist: ifelse(o.ipV6, allowInternalAllowIPv6, allowInternalProhibitIPv6), + notOnWhitelist: ifelse(o.ipV6, prohibitInternalAllowIPv6, prohibitInternalProhibitIPv6), + internalIPExceptions: o.internalIPExceptions, + } + } else { + o.c.Transport = ifelse(o.ipV6, allowInternalAllowIPv6, allowInternalProhibitIPv6) } if o.tracer != nil { - o.c.Transport = otelhttp.NewTransport(o.c.Transport) + o.c.Transport = otelhttp.NewTransport(o.c.Transport, otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutHeaders(), otelhttptrace.WithoutSubSpans()) + })) } cl := retryablehttp.NewClient() @@ -146,3 +164,10 @@ func SetOAuth2(ctx context.Context, cl *retryablehttp.Client, c OAuth2Config, t type OAuth2Config interface { Client(context.Context, *oauth2.Token) *http.Client } + +func ifelse[A any](b bool, x, y A) A { + if b { + return x + } + return y +} diff --git a/httpx/resilient_client_test.go b/httpx/resilient_client_test.go index 917dfc85..8a90118b 100644 --- a/httpx/resilient_client_test.go +++ b/httpx/resilient_client_test.go @@ -4,12 +4,17 @@ package httpx import ( + "context" "net" "net/http" "net/http/httptest" + "net/http/httptrace" + "net/netip" "net/url" + "sync/atomic" "testing" + "github.com/hashicorp/go-retryablehttp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -55,3 +60,71 @@ func TestNoPrivateIPs(t *testing.T) { } } } + +func TestNoIPV6(t *testing.T) { + for _, tc := range []struct { + name string + c *retryablehttp.Client + }{ + { + "internal IPs allowed", + NewResilientClient( + ResilientClientWithMaxRetry(1), + ResilientClientNoIPv6(), + ), + }, { + "internal IPs disallowed", + NewResilientClient( + ResilientClientWithMaxRetry(1), + ResilientClientDisallowInternalIPs(), + ResilientClientNoIPv6(), + ), + }, + } { + t.Run(tc.name, func(t *testing.T) { + var connectDone int32 + ctx := httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ + DNSDone: func(dnsInfo httptrace.DNSDoneInfo) { + for _, ip := range dnsInfo.Addrs { + netIP, ok := netip.AddrFromSlice(ip.IP) + assert.True(t, ok) + assert.Truef(t, netIP.Is4(), "ip = %s", ip) + } + }, + ConnectDone: func(network, addr string, err error) { + atomic.AddInt32(&connectDone, 1) + assert.NoError(t, err) + assert.Equalf(t, "tcp4", network, "network = %s addr = %s", network, addr) + }, + }) + + // Dual stack + req, err := retryablehttp.NewRequestWithContext(ctx, "GET", "http://dual.tlund.se/", nil) + require.NoError(t, err) + atomic.StoreInt32(&connectDone, 0) + res, err := tc.c.Do(req) + require.GreaterOrEqual(t, int32(1), atomic.LoadInt32(&connectDone)) + require.NoError(t, err) + t.Cleanup(func() { _ = res.Body.Close() }) + require.EqualValues(t, http.StatusOK, res.StatusCode) + + // IPv4 only + req, err = retryablehttp.NewRequestWithContext(ctx, "GET", "http://ipv4.tlund.se/", nil) + require.NoError(t, err) + atomic.StoreInt32(&connectDone, 0) + res, err = tc.c.Do(req) + require.EqualValues(t, 1, atomic.LoadInt32(&connectDone)) + require.NoError(t, err) + t.Cleanup(func() { _ = res.Body.Close() }) + require.EqualValues(t, http.StatusOK, res.StatusCode) + + // IPv6 only + req, err = retryablehttp.NewRequestWithContext(ctx, "GET", "http://ipv6.tlund.se/", nil) + require.NoError(t, err) + atomic.StoreInt32(&connectDone, 0) + _, err = tc.c.Do(req) + require.EqualValues(t, 0, atomic.LoadInt32(&connectDone)) + require.ErrorContains(t, err, "no such host") + }) + } +} diff --git a/httpx/ssrf.go b/httpx/ssrf.go index 6d81cc72..b92d579e 100644 --- a/httpx/ssrf.go +++ b/httpx/ssrf.go @@ -4,6 +4,7 @@ package httpx import ( + "context" "net" "net/http" "net/netip" @@ -23,41 +24,12 @@ type noInternalIPRoundTripper struct { // NewNoInternalIPRoundTripper creates a RoundTripper that disallows // non-publicly routable IP addresses, except for URLs matching the given // exception globs. +// Deprecated: Use ResilientClientDisallowInternalIPs instead. func NewNoInternalIPRoundTripper(exceptions []string) http.RoundTripper { - if len(exceptions) > 0 { - prohibitInternal := newSSRFTransport(ssrf.New( - ssrf.WithAnyPort(), - ssrf.WithNetworks("tcp4", "tcp6"), - )) - - allowInternal := newSSRFTransport(ssrf.New( - ssrf.WithAnyPort(), - ssrf.WithNetworks("tcp4", "tcp6"), - ssrf.WithAllowedV4Prefixes( - netip.MustParsePrefix("10.0.0.0/8"), // Private-Use (RFC 1918) - netip.MustParsePrefix("127.0.0.0/8"), // Loopback (RFC 1122, Section 3.2.1.3)) - netip.MustParsePrefix("169.254.0.0/16"), // Link Local (RFC 3927) - netip.MustParsePrefix("172.16.0.0/12"), // Private-Use (RFC 1918) - netip.MustParsePrefix("192.168.0.0/16"), // Private-Use (RFC 1918) - ), - ssrf.WithAllowedV6Prefixes( - netip.MustParsePrefix("::1/128"), // Loopback (RFC 4193) - netip.MustParsePrefix("fc00::/7"), // Unique Local (RFC 4193) - ), - )) - return noInternalIPRoundTripper{ - onWhitelist: allowInternal, - notOnWhitelist: prohibitInternal, - internalIPExceptions: exceptions, - } - } - prohibitInternal := newSSRFTransport(ssrf.New( - ssrf.WithAnyPort(), - ssrf.WithNetworks("tcp4", "tcp6"), - )) - return noInternalIPRoundTripper{ - onWhitelist: prohibitInternal, - notOnWhitelist: prohibitInternal, + return &noInternalIPRoundTripper{ + onWhitelist: allowInternalAllowIPv6, + notOnWhitelist: prohibitInternalAllowIPv6, + internalIPExceptions: exceptions, } } @@ -79,23 +51,89 @@ func (n noInternalIPRoundTripper) RoundTrip(request *http.Request) (*http.Respon return n.notOnWhitelist.RoundTrip(request) } -func newSSRFTransport(g *ssrf.Guardian) http.RoundTripper { - t := newDefaultTransport() - t.DialContext = (&net.Dialer{Control: g.Safe}).DialContext - return t +var ( + prohibitInternalAllowIPv6 http.RoundTripper + prohibitInternalProhibitIPv6 http.RoundTripper + allowInternalAllowIPv6 http.RoundTripper + allowInternalProhibitIPv6 http.RoundTripper +) + +func init() { + t, d := newDefaultTransport() + d.Control = ssrf.New( + ssrf.WithAnyPort(), + ssrf.WithNetworks("tcp4", "tcp6"), + ).Safe + prohibitInternalAllowIPv6 = t } -func newDefaultTransport() *http.Transport { +func init() { + t, d := newDefaultTransport() + d.Control = ssrf.New( + ssrf.WithAnyPort(), + ssrf.WithNetworks("tcp4"), + ).Safe + t.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + return d.DialContext(ctx, "tcp4", addr) + } + prohibitInternalProhibitIPv6 = t +} + +func init() { + t, d := newDefaultTransport() + d.Control = ssrf.New( + ssrf.WithAnyPort(), + ssrf.WithNetworks("tcp4", "tcp6"), + ssrf.WithAllowedV4Prefixes( + netip.MustParsePrefix("10.0.0.0/8"), // Private-Use (RFC 1918) + netip.MustParsePrefix("127.0.0.0/8"), // Loopback (RFC 1122, Section 3.2.1.3)) + netip.MustParsePrefix("169.254.0.0/16"), // Link Local (RFC 3927) + netip.MustParsePrefix("172.16.0.0/12"), // Private-Use (RFC 1918) + netip.MustParsePrefix("192.168.0.0/16"), // Private-Use (RFC 1918) + ), + ssrf.WithAllowedV6Prefixes( + netip.MustParsePrefix("::1/128"), // Loopback (RFC 4193) + netip.MustParsePrefix("fc00::/7"), // Unique Local (RFC 4193) + ), + ).Safe + allowInternalAllowIPv6 = t +} + +func init() { + t, d := newDefaultTransport() + d.Control = ssrf.New( + ssrf.WithAnyPort(), + ssrf.WithNetworks("tcp4"), + ssrf.WithAllowedV4Prefixes( + netip.MustParsePrefix("10.0.0.0/8"), // Private-Use (RFC 1918) + netip.MustParsePrefix("127.0.0.0/8"), // Loopback (RFC 1122, Section 3.2.1.3)) + netip.MustParsePrefix("169.254.0.0/16"), // Link Local (RFC 3927) + netip.MustParsePrefix("172.16.0.0/12"), // Private-Use (RFC 1918) + netip.MustParsePrefix("192.168.0.0/16"), // Private-Use (RFC 1918) + ), + ssrf.WithAllowedV6Prefixes( + netip.MustParsePrefix("::1/128"), // Loopback (RFC 4193) + netip.MustParsePrefix("fc00::/7"), // Unique Local (RFC 4193) + ), + ).Safe + t.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + return d.DialContext(ctx, "tcp4", addr) + } + allowInternalProhibitIPv6 = t +} + +func newDefaultTransport() (*http.Transport, *net.Dialer) { + dialer := net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + } return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, + Proxy: http.ProxyFromEnvironment, + DialContext: dialer.DialContext, ForceAttemptHTTP2: true, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, - } + }, &dialer } diff --git a/jsonnetsecure/jsonnet_pool.go b/jsonnetsecure/jsonnet_pool.go index 3447bbff..f89abd49 100644 --- a/jsonnetsecure/jsonnet_pool.go +++ b/jsonnetsecure/jsonnet_pool.go @@ -156,7 +156,8 @@ func (w worker) destroy() { func (w worker) eval(ctx context.Context, processParams []byte) (output string, err error) { tracer := trace.SpanFromContext(ctx).TracerProvider().Tracer("") - ctx, span := tracer.Start(ctx, "jsonnetsecure.worker.eval") + ctx, span := tracer.Start(ctx, "jsonnetsecure.worker.eval", + trace.WithAttributes(attribute.Int("cmd.Process.Pid", w.cmd.Process.Pid))) defer otelx.End(span, &err) select {