Skip to content

Commit

Permalink
move k6catalog into k6build
Browse files Browse the repository at this point in the history
Signed-off-by: Pablo Chacin <[email protected]>
  • Loading branch information
pablochacin committed Jan 10, 2025
1 parent f53b4e7 commit 92330ca
Show file tree
Hide file tree
Showing 10 changed files with 431 additions and 21 deletions.
4 changes: 2 additions & 2 deletions cmd/local/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import (
"strings"

"github.com/grafana/k6build"
"github.com/grafana/k6build/pkg/catalog"
"github.com/grafana/k6build/pkg/local"
"github.com/grafana/k6catalog"

"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -125,7 +125,7 @@ func New() *cobra.Command { //nolint:funlen
cmd.Flags().StringVarP(&k6, "k6", "k", "*", "k6 version constrains")
cmd.Flags().StringVarP(&platform, "platform", "p", "", "target platform (default GOOS/GOARCH)")
_ = cmd.MarkFlagRequired("platform")
cmd.Flags().StringVarP(&config.Catalog, "catalog", "c", k6catalog.DefaultCatalogURL, "dependencies catalog")
cmd.Flags().StringVarP(&config.Catalog, "catalog", "c", catalog.DefaultCatalogURL, "dependencies catalog")
cmd.Flags().StringVarP(&config.StoreDir, "store-dir", "f", "/tmp/k6build/store", "object store dir")
cmd.Flags().BoolVarP(&config.Opts.Verbose, "verbose", "v", false, "print build process output")
cmd.Flags().BoolVarP(&config.CopyGoEnv, "copy-go-env", "g", true, "copy go environment")
Expand Down
6 changes: 3 additions & 3 deletions cmd/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ import (

"github.com/grafana/k6build"
"github.com/grafana/k6build/pkg/builder"
"github.com/grafana/k6build/pkg/catalog"
"github.com/grafana/k6build/pkg/server"
"github.com/grafana/k6build/pkg/store"
"github.com/grafana/k6build/pkg/store/client"
"github.com/grafana/k6build/pkg/store/s3"
"github.com/grafana/k6catalog"

"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -116,7 +116,7 @@ func New() *cobra.Command { //nolint:funlen
),
)

catalog, err := k6catalog.NewCatalog(cmd.Context(), catalogURL)
catalog, err := catalog.NewCatalog(cmd.Context(), catalogURL)
if err != nil {
return fmt.Errorf("creating catalog %w", err)
}
Expand Down Expand Up @@ -193,7 +193,7 @@ func New() *cobra.Command { //nolint:funlen
&catalogURL,
"catalog",
"c",
k6catalog.DefaultCatalogURL,
catalog.DefaultCatalogURL,
"dependencies catalog. Can be path to a local file or an URL."+
"\n",
)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/grafana/k6build
go 1.22.2

require (
github.com/Masterminds/semver/v3 v3.3.1
github.com/aws/aws-sdk-go-v2 v1.32.7
github.com/aws/aws-sdk-go-v2/config v1.28.6
github.com/aws/aws-sdk-go-v2/credentials v1.17.47
Expand All @@ -18,7 +19,6 @@ require (
require (
dario.cat/mergo v1.0.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/Masterminds/semver/v3 v3.3.1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
Expand Down
14 changes: 7 additions & 7 deletions pkg/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import (
"sync"

"github.com/grafana/k6build"
"github.com/grafana/k6build/pkg/catalog"
"github.com/grafana/k6build/pkg/store"
"github.com/grafana/k6catalog"
"github.com/grafana/k6foundry"
)

Expand Down Expand Up @@ -53,14 +53,14 @@ type Opts struct {
// Config defines the configuration for a Builder
type Config struct {
Opts Opts
Catalog k6catalog.Catalog
Catalog catalog.Catalog
Store store.ObjectStore
}

// Builder implements the BuildService interface
type Builder struct {
allowBuildSemvers bool
catalog k6catalog.Catalog
catalog catalog.Catalog
builder k6foundry.Builder
store store.ObjectStore
mutexes sync.Map
Expand Down Expand Up @@ -121,7 +121,7 @@ func (b *Builder) Build( //nolint:funlen
// the build metadata as version when building this module
// the build process will return the actual version built in the build info
// and we can check that version with the catalog
var k6Mod k6catalog.Module
var k6Mod catalog.Module
buildMetadata, err := hasBuildMetadata(k6Constrains)
if err != nil {
return k6build.Artifact{}, err
Expand All @@ -130,9 +130,9 @@ func (b *Builder) Build( //nolint:funlen
if !b.allowBuildSemvers {
return k6build.Artifact{}, k6build.NewWrappedError(ErrInvalidParameters, ErrBuildSemverNotAllowed)
}
k6Mod = k6catalog.Module{Path: k6Path, Version: buildMetadata}
k6Mod = catalog.Module{Path: k6Path, Version: buildMetadata}
} else {
k6Mod, err = b.catalog.Resolve(ctx, k6catalog.Dependency{Name: k6Dep, Constrains: k6Constrains})
k6Mod, err = b.catalog.Resolve(ctx, catalog.Dependency{Name: k6Dep, Constrains: k6Constrains})
if err != nil {
return k6build.Artifact{}, k6build.NewWrappedError(ErrInvalidParameters, err)
}
Expand All @@ -141,7 +141,7 @@ func (b *Builder) Build( //nolint:funlen

mods := []k6foundry.Module{}
for _, d := range deps {
m, modErr := b.catalog.Resolve(ctx, k6catalog.Dependency{Name: d.Name, Constrains: d.Constraints})
m, modErr := b.catalog.Resolve(ctx, catalog.Dependency{Name: d.Name, Constrains: d.Constraints})
if modErr != nil {
return k6build.Artifact{}, k6build.NewWrappedError(ErrInvalidParameters, modErr)
}
Expand Down
200 changes: 200 additions & 0 deletions pkg/catalog/catalog.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
// Package catalog defines the extension catalog
//
// A catalog maps a dependency for k6 extension with optional semantic versioning
// constrains to the corresponding golang modules.
//
// For example `k6/x/output-kafka:>0.1.0` ==> `github.com/grafana/[email protected]`
//
// The catalog is a json file with the following schema:
//
// {
// "<dependency>": {"module": "<go module path>", "versions": ["<version>", "<version>", ... "<version>"]}
// }
//
// where:
// <dependency>: is the import path for the dependency
// module: is the path to the go module that implements the dependency
// versions: is the list of supported versions
//
// Example:
//
// {
// "k6": {"module": "go.k6.io/k6", "versions": ["v0.50.0", "v0.51.0"]},
// "k6/x/kubernetes": {"module": "github.com/grafana/xk6-kubernetes", "versions": ["v0.8.0","v0.9.0"]},
// "k6/x/output-kafka": {"module": "github.com/grafana/xk6-output-kafka", "versions": ["v0.7.0"]}
// }
package catalog

import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"strings"

"github.com/Masterminds/semver/v3"
)

const (
DefaultCatalogFile = "catalog.json" //nolint:revive
DefaultCatalogURL = "https://registry.k6.io/catalog.json" //nolint:revive
)

var (
ErrCannotSatisfy = errors.New("cannot satisfy dependency") //nolint:revive
ErrDownload = errors.New("downloading catalog") //nolint:revive
ErrInvalidConstrain = errors.New("invalid constrain") //nolint:revive
ErrInvalidCatalog = fmt.Errorf("invalid catalog") //nolint:revive
ErrOpening = errors.New("opening catalog") //nolint:revive
ErrUnknownDependency = errors.New("unknown dependency") //nolint:revive

)

// Dependency defines a Dependency with a version constrain
// Examples:
// Name: k6/x/k6-kubernetes Constrains *
// Name: k6/x/k6-output-kafka Constrains >v0.9.0
type Dependency struct {
Name string `json:"name,omitempty"`
Constrains string `json:"constrains,omitempty"`
}

// Module defines a go module that resolves a Dependency
type Module struct {
Path string `json:"path,omitempty"`
Version string `json:"version,omitempty"`
}

// Catalog defines the interface of the extension catalog service
type Catalog interface {
// Resolve returns a Module that satisfies a Dependency
Resolve(ctx context.Context, dep Dependency) (Module, error)
}

// entry defines a catalog entry
type entry struct {
Module string `json:"module,omitempty"`
Versions []string `json:"versions,omitempty"`
}

type catalog struct {
dependencies map[string]entry
}

// getVersions returns the versions for a given module
func (c catalog) getVersions(_ context.Context, mod string) (entry, error) {
e, found := c.dependencies[mod]
if !found {
return entry{}, fmt.Errorf("%w : %s", ErrUnknownDependency, mod)
}

return e, nil
}

// NewCatalogFromJSON creates a Catalog from a json file that follows the [schema](./schema.json):
func NewCatalogFromJSON(stream io.Reader) (Catalog, error) {
buff := &bytes.Buffer{}
_, err := buff.ReadFrom(stream)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidCatalog, err)
}

dependencies := map[string]entry{}
err = json.Unmarshal(buff.Bytes(), &dependencies)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidCatalog, err)
}

return catalog{
dependencies: dependencies,
}, nil
}

// NewCatalog returns a catalog loaded from a location.
// The location can be a local path or an URL
func NewCatalog(ctx context.Context, location string) (Catalog, error) {
if strings.HasPrefix(location, "http") {
return NewCatalogFromURL(ctx, location)
}

return NewCatalogFromFile(location)
}

// NewCatalogFromFile creates a Catalog from a json file
func NewCatalogFromFile(catalogFile string) (Catalog, error) {
json, err := os.ReadFile(catalogFile) //nolint:gosec
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrOpening, err)
}

buff := bytes.NewBuffer(json)
return NewCatalogFromJSON(buff)
}

// NewCatalogFromURL creates a Catalog from a URL
func NewCatalogFromURL(ctx context.Context, catalogURL string) (Catalog, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, catalogURL, nil)
if err != nil {
return nil, fmt.Errorf("%w %w", ErrDownload, err)
}

resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("%w %w", ErrDownload, err)
}
defer resp.Body.Close() //nolint:errcheck

if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w %s", ErrDownload, resp.Status)
}

catalog, err := NewCatalogFromJSON(resp.Body)
if err != nil {
return nil, fmt.Errorf("%w %w", ErrDownload, err)
}

return catalog, nil
}

// DefaultCatalog creates a Catalog from the default catalog URL
func DefaultCatalog() (Catalog, error) {
return NewCatalogFromURL(context.TODO(), DefaultCatalogURL)
}

func (c catalog) Resolve(ctx context.Context, dep Dependency) (Module, error) {
entry, err := c.getVersions(ctx, dep.Name)
if err != nil {
return Module{}, err
}

constrain, err := semver.NewConstraint(dep.Constrains)
if err != nil {
return Module{}, fmt.Errorf("%w : %s", ErrInvalidConstrain, dep.Constrains)
}

versions := []*semver.Version{}
for _, v := range entry.Versions {
version, err := semver.NewVersion(v)
if err != nil {
return Module{}, err
}
versions = append(versions, version)
}

if len(versions) > 0 {
// try to find the higher version that satisfies the condition
sort.Sort(sort.Reverse(semver.Collection(versions)))
for _, v := range versions {
if constrain.Check(v) {
return Module{Path: entry.Module, Version: v.Original()}, nil
}
}
}

return Module{}, fmt.Errorf("%w : %s %s", ErrCannotSatisfy, dep.Name, dep.Constrains)
}
Loading

0 comments on commit 92330ca

Please sign in to comment.