diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05e5004b6a..7f4cf69f50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -383,7 +383,7 @@ jobs: - name: Upload artifacts if: env.run_tests - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: e2e-artifact path: builddir/e2e-cmd-report.txt diff --git a/LICENSE_DEPENDENCIES.md b/LICENSE_DEPENDENCIES.md index 36316714ad..f799a2ac6a 100644 --- a/LICENSE_DEPENDENCIES.md +++ b/LICENSE_DEPENDENCIES.md @@ -917,12 +917,6 @@ The dependencies and their licenses are as follows: **License URL:** -## github.com/urfave/cli - -**License:** MIT - -**License URL:** - ## go.mozilla.org/pkcs7 **License:** MIT diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index c42894dfd6..c06be69436 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -27,7 +27,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/client/oras" "github.com/apptainer/apptainer/internal/pkg/client/shub" "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/ocitransport" + "github.com/apptainer/apptainer/internal/pkg/ociimage" "github.com/apptainer/apptainer/internal/pkg/runtime/launch" "github.com/apptainer/apptainer/internal/pkg/util/env" "github.com/apptainer/apptainer/internal/pkg/util/uri" @@ -158,7 +158,7 @@ func replaceURIWithImage(ctx context.Context, cmd *cobra.Command, args []string) image, err = handleOras(ctx, imgCache, cmd, args[0]) case uri.Shub: image, err = handleShub(ctx, imgCache, args[0]) - case ocitransport.SupportedTransport(t): + case ociimage.SupportedTransport(t): image, err = handleOCI(ctx, imgCache, cmd, args[0]) case uri.HTTP: image, err = handleNet(ctx, imgCache, args[0]) diff --git a/cmd/internal/cli/pull.go b/cmd/internal/cli/pull.go index b29c82cf71..db682f8128 100644 --- a/cmd/internal/cli/pull.go +++ b/cmd/internal/cli/pull.go @@ -23,7 +23,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/client/oci" "github.com/apptainer/apptainer/internal/pkg/client/oras" "github.com/apptainer/apptainer/internal/pkg/client/shub" - "github.com/apptainer/apptainer/internal/pkg/ocitransport" + "github.com/apptainer/apptainer/internal/pkg/ociimage" "github.com/apptainer/apptainer/internal/pkg/remote/endpoint" "github.com/apptainer/apptainer/internal/pkg/util/uri" "github.com/apptainer/apptainer/pkg/cmdline" @@ -278,7 +278,7 @@ func pullRun(cmd *cobra.Command, args []string) { if err != nil { sylog.Fatalf("While pulling from image from http(s): %v\n", err) } - case ocitransport.SupportedTransport(transport): + case ociimage.SupportedTransport(transport): ociAuth, err := makeOCICredentials(cmd) if err != nil { sylog.Fatalf("While creating Docker credentials: %v", err) diff --git a/go.mod b/go.mod index 8e795a8de1..e7231016fb 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,6 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/sylabs/json-resp v0.9.3 - github.com/urfave/cli v1.22.14 // indirect github.com/vbauerster/mpb/v8 v8.8.3 golang.org/x/crypto v0.26.0 golang.org/x/sys v0.24.0 diff --git a/go.sum b/go.sum index 4f4cd503a1..7f752948e8 100644 --- a/go.sum +++ b/go.sum @@ -9,7 +9,6 @@ github.com/AdamKorcz/go-fuzz-headers v0.0.0-20210319161527-f761c2329661/go.mod h github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -94,7 +93,6 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -481,8 +479,6 @@ github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= -github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/vbatts/go-mtree v0.5.0 h1:dM+5XZdqH0j9CSZeerhoN/tAySdwnmevaZHO1XGW2Vc= github.com/vbatts/go-mtree v0.5.0/go.mod h1:7JbaNHyBMng+RP8C3Q4E+4Ca8JnGQA2R/MB+jb4tSOk= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= diff --git a/internal/pkg/build/conveyorPacker.go b/internal/pkg/build/conveyorPacker.go index 185753f0b8..9d0e89e25d 100644 --- a/internal/pkg/build/conveyorPacker.go +++ b/internal/pkg/build/conveyorPacker.go @@ -14,7 +14,7 @@ import ( "fmt" "github.com/apptainer/apptainer/internal/pkg/build/sources" - "github.com/apptainer/apptainer/internal/pkg/ocitransport" + "github.com/apptainer/apptainer/internal/pkg/ociimage" "github.com/apptainer/apptainer/pkg/build/types" ) @@ -48,7 +48,7 @@ func conveyorPacker(def types.Definition) (ConveyorPacker, error) { return &sources.OrasConveyorPacker{}, nil case "shub": return &sources.ShubConveyorPacker{}, nil - case ocitransport.SupportedTransport(bs): + case ociimage.SupportedTransport(bs): return &sources.OCIConveyorPacker{}, nil case "busybox": return &sources.BusyBoxConveyorPacker{}, nil diff --git a/internal/pkg/build/oci/oci.go b/internal/pkg/build/oci/oci.go index b39a84ecde..2900ecb411 100644 --- a/internal/pkg/build/oci/oci.go +++ b/internal/pkg/build/oci/oci.go @@ -19,7 +19,7 @@ import ( "strings" "github.com/apptainer/apptainer/internal/pkg/cache" - "github.com/apptainer/apptainer/internal/pkg/ocitransport" + "github.com/apptainer/apptainer/internal/pkg/ociimage" "github.com/apptainer/apptainer/pkg/sylog" "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker" @@ -76,14 +76,14 @@ var ArchMap = map[string]GoArch{ } // ConvertReference converts a source reference into a cache.ImageReference to cache its blobs -func ConvertReference(ctx context.Context, imgCache *cache.Handle, src types.ImageReference, topts *ocitransport.TransportOptions) (types.ImageReference, error) { +func ConvertReference(ctx context.Context, imgCache *cache.Handle, src types.ImageReference, topts *ociimage.TransportOptions) (types.ImageReference, error) { if imgCache == nil { return nil, fmt.Errorf("undefined image cache") } if topts == nil { // nolint:staticcheck - topts = ocitransport.TransportOptionsFromSystemContext(nil) + topts = ociimage.TransportOptionsFromSystemContext(nil) } // Our cache dir is an OCI directory. We are using this as a 'blob pool' @@ -135,7 +135,7 @@ func (t *ImageReference) newImageSource(ctx context.Context, sys *types.SystemCo // ParseImageName parses a uri (e.g. docker://ubuntu) into it's transport:reference // combination and then returns the proper reference -func ParseImageName(ctx context.Context, imgCache *cache.Handle, uri string, topts *ocitransport.TransportOptions) (types.ImageReference, error) { +func ParseImageName(ctx context.Context, imgCache *cache.Handle, uri string, topts *ociimage.TransportOptions) (types.ImageReference, error) { ref, _, err := parseURI(uri) if err != nil { return nil, fmt.Errorf("unable to parse image name %v: %v", uri, err) @@ -164,7 +164,7 @@ func parseURI(uri string) (types.ImageReference, *GoArch, error) { } // ImageDigest obtains the digest of a uri's manifest -func ImageDigest(ctx context.Context, uri string, topts *ocitransport.TransportOptions) (digest string, err error) { +func ImageDigest(ctx context.Context, uri string, topts *ociimage.TransportOptions) (digest string, err error) { ref, arch, err := parseURI(uri) if err != nil { return "", fmt.Errorf("unable to parse image name %v: %v", uri, err) @@ -179,7 +179,7 @@ func ImageDigest(ctx context.Context, uri string, topts *ocitransport.TransportO } // getRefDigest obtains the manifest digest for a ref. -func getRefDigest(ctx context.Context, ref types.ImageReference, topts *ocitransport.TransportOptions) (digest string, err error) { +func getRefDigest(ctx context.Context, ref types.ImageReference, topts *ociimage.TransportOptions) (digest string, err error) { // Handle docker references specially, using a HEAD request to ensure we don't hit API limits if ref.Transport().Name() == "docker" { digest, err := getDockerRefDigest(ctx, ref, topts) @@ -194,7 +194,7 @@ func getRefDigest(ctx context.Context, ref types.ImageReference, topts *ocitrans // Otherwise get the manifest and calculate sha256 over it // nolint:staticcheck - source, err := ref.NewImageSource(ctx, ocitransport.SystemContextFromTransportOptions(topts)) + source, err := ref.NewImageSource(ctx, ociimage.SystemContextFromTransportOptions(topts)) if err != nil { return "", err } @@ -216,9 +216,9 @@ func getRefDigest(ctx context.Context, ref types.ImageReference, topts *ocitrans } // getDockerRefDigest obtains the manifest digest for a docker ref. -func getDockerRefDigest(ctx context.Context, ref types.ImageReference, topts *ocitransport.TransportOptions) (digest string, err error) { +func getDockerRefDigest(ctx context.Context, ref types.ImageReference, topts *ociimage.TransportOptions) (digest string, err error) { // nolint:staticcheck - d, err := docker.GetDigest(ctx, ocitransport.SystemContextFromTransportOptions(topts), ref) + d, err := docker.GetDigest(ctx, ociimage.SystemContextFromTransportOptions(topts), ref) if err != nil { return "", err } diff --git a/internal/pkg/build/oci/oci_test.go b/internal/pkg/build/oci/oci_test.go index c83a8faca5..2054dd2bc4 100644 --- a/internal/pkg/build/oci/oci_test.go +++ b/internal/pkg/build/oci/oci_test.go @@ -20,7 +20,7 @@ import ( "testing" "github.com/apptainer/apptainer/internal/pkg/cache" - "github.com/apptainer/apptainer/internal/pkg/ocitransport" + "github.com/apptainer/apptainer/internal/pkg/ociimage" "github.com/apptainer/apptainer/internal/pkg/test" buildTypes "github.com/apptainer/apptainer/pkg/build/types" "github.com/containers/image/v5/oci/layout" @@ -275,7 +275,7 @@ func TestConvertReference(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // nolint: staticcheck - _, err := ConvertReference(context.Background(), imgCache, tt.ref, ocitransport.TransportOptionsFromSystemContext(tt.ctx)) + _, err := ConvertReference(context.Background(), imgCache, tt.ref, ociimage.TransportOptionsFromSystemContext(tt.ctx)) if tt.shouldPass == true && err != nil { t.Fatalf("test expected to succeeded but failed: %s\n", err) } @@ -347,7 +347,7 @@ func TestImageNameAndImageSHA(t *testing.T) { testName := "ParseImageName - " + tt.name t.Run(testName, func(t *testing.T) { // nolint:staticcheck - _, err := ParseImageName(context.Background(), imgCache, tt.uri, ocitransport.TransportOptionsFromSystemContext(tt.ctx)) + _, err := ParseImageName(context.Background(), imgCache, tt.uri, ociimage.TransportOptionsFromSystemContext(tt.ctx)) if tt.shouldPass == true && err != nil { t.Fatalf("test expected to succeeded but failed: %s\n", err) } @@ -359,7 +359,7 @@ func TestImageNameAndImageSHA(t *testing.T) { testName = "ImageSHA - " + tt.name t.Run(testName, func(t *testing.T) { // nolint: staticcheck - _, err := ImageDigest(context.Background(), tt.uri, ocitransport.TransportOptionsFromSystemContext(tt.ctx)) + _, err := ImageDigest(context.Background(), tt.uri, ociimage.TransportOptionsFromSystemContext(tt.ctx)) if tt.shouldPass == true && err != nil { t.Fatal("test expected to succeeded but failed") } diff --git a/internal/pkg/build/sources/conveyorPacker_oci.go b/internal/pkg/build/sources/conveyorPacker_oci.go index 34e997d671..585e4b74a5 100644 --- a/internal/pkg/build/sources/conveyorPacker_oci.go +++ b/internal/pkg/build/sources/conveyorPacker_oci.go @@ -11,39 +11,26 @@ package sources import ( - "archive/tar" - "bufio" "bytes" - "compress/gzip" "context" "encoding/json" "fmt" - "io" - "net/http" "os" "path/filepath" - "reflect" "strings" "text/template" - "github.com/apptainer/apptainer/internal/pkg/build/oci" - "github.com/apptainer/apptainer/internal/pkg/ocitransport" + "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/ociimage" + "github.com/apptainer/apptainer/internal/pkg/ociplatform" "github.com/apptainer/apptainer/internal/pkg/util/ociauth" "github.com/apptainer/apptainer/internal/pkg/util/shell" sytypes "github.com/apptainer/apptainer/pkg/build/types" "github.com/apptainer/apptainer/pkg/image" "github.com/apptainer/apptainer/pkg/sylog" useragent "github.com/apptainer/apptainer/pkg/util/user-agent" - "github.com/containers/image/v5/copy" - "github.com/containers/image/v5/docker" - dockerarchive "github.com/containers/image/v5/docker/archive" - dockerdaemon "github.com/containers/image/v5/docker/daemon" - ociarchive "github.com/containers/image/v5/oci/archive" - ocilayout "github.com/containers/image/v5/oci/layout" - "github.com/containers/image/v5/signature" - "github.com/containers/image/v5/types" + "github.com/google/go-containerregistry/pkg/authn" v1 "github.com/google/go-containerregistry/pkg/v1" - imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) type ociRunscriptData struct { @@ -133,30 +120,17 @@ exec "$@" // OCIConveyorPacker holds stuff that needs to be packed into the bundle type OCIConveyorPacker struct { - srcRef types.ImageReference + srcImg v1.Image b *sytypes.Bundle - tmpfsRef types.ImageReference - policyCtx *signature.PolicyContext - imgConfig imgspecv1.ImageConfig - topts *ocitransport.TransportOptions + imgConfig v1.Config + topts *ociimage.TransportOptions } // Get downloads container information from the specified source func (cp *OCIConveyorPacker) Get(ctx context.Context, b *sytypes.Bundle) (err error) { cp.b = b - policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} - cp.policyCtx, err = signature.NewPolicyContext(policy) - if err != nil { - return err - } - - // DockerInsecureSkipTLSVerify is set only if --no-https is specified to honor - // configuration from /etc/containers/registries.conf because DockerInsecureSkipTLSVerify - // can have three possible values true/false and undefined, so we left it as undefined instead - // of forcing it to false in order to delegate decision to /etc/containers/registries.conf: - // https://github.com/apptainer/singularity/issues/5172 - cp.topts = &ocitransport.TransportOptions{ + cp.topts = &ociimage.TransportOptions{ Insecure: cp.b.Opts.NoHTTPS, DockerDaemonHost: cp.b.Opts.DockerDaemonHost, AuthConfig: cp.b.Opts.OCIAuthConfig, @@ -165,19 +139,21 @@ func (cp *OCIConveyorPacker) Get(ctx context.Context, b *sytypes.Bundle) (err er TmpDir: b.TmpDir, } - if cp.b.Opts.Arch != "" { - if arch, ok := oci.ArchMap[cp.b.Opts.Arch]; ok { - cp.topts.Platform = v1.Platform{ - Architecture: arch.Arch, - Variant: arch.Var, - } - } else { - keys := reflect.ValueOf(oci.ArchMap).MapKeys() - return fmt.Errorf("failed to parse the arch value: %s, should be one of %v", cp.b.Opts.Arch, keys) + if cp.b.Opts.OCIAuthConfig == nil && cp.b.Opts.DockerAuthConfig != nil { + cp.topts.AuthConfig = &authn.AuthConfig{ + Username: cp.b.Opts.DockerAuthConfig.Username, + Password: cp.b.Opts.DockerAuthConfig.Password, + IdentityToken: cp.b.Opts.DockerAuthConfig.IdentityToken, } } - // add registry and namespace to reference if specified + dp, err := ociplatform.DefaultPlatform() + if err != nil { + return err + } + cp.topts.Platform = *dp + + // Add registry and namespace to image reference if specified ref := b.Recipe.Header["from"] if b.Recipe.Header["namespace"] != "" { ref = b.Recipe.Header["namespace"] + "/" + ref @@ -185,88 +161,38 @@ func (cp *OCIConveyorPacker) Get(ctx context.Context, b *sytypes.Bundle) (err er if b.Recipe.Header["registry"] != "" { ref = b.Recipe.Header["registry"] + "/" + ref } - sylog.Debugf("Reference: %v", ref) - - switch b.Recipe.Header["bootstrap"] { - case "docker": + // Docker sources are docker://, not docker: + if b.Recipe.Header["bootstrap"] == "docker" { ref = "//" + ref - cp.srcRef, err = docker.ParseReference(ref) - case "docker-archive": - cp.srcRef, err = dockerarchive.ParseReference(ref) - case "docker-daemon": - cp.srcRef, err = dockerdaemon.ParseReference(ref) - case "oci": - cp.srcRef, err = ocilayout.ParseReference(ref) - case "oci-archive": - if os.Geteuid() == 0 { - // As root, the direct oci-archive handling will work - cp.srcRef, err = ociarchive.ParseReference(ref) - } else { - // As non-root we need to do a dumb tar extraction first - tmpDir, err := os.MkdirTemp(b.TmpDir, "temp-oci-") - if err != nil { - return fmt.Errorf("could not create temporary oci directory: %v", err) - } - defer os.RemoveAll(tmpDir) - - refParts := strings.SplitN(b.Recipe.Header["from"], ":", 2) - err = cp.extractArchive(refParts[0], tmpDir) - if err != nil { - return fmt.Errorf("error extracting the OCI archive file: %v", err) - } - // We may or may not have had a ':tag' in the source to handle - if len(refParts) == 2 { - cp.srcRef, err = ocilayout.ParseReference(tmpDir + ":" + refParts[1]) - } else { - cp.srcRef, err = ocilayout.ParseReference(tmpDir) - } - - if err != nil { - return fmt.Errorf("error parsing reference: %v", err) - } - } - - default: - return fmt.Errorf("oci conveyorPacker does not support %s", b.Recipe.Header["bootstrap"]) - } - - if err != nil { - return fmt.Errorf("invalid image source: %v", err) } + // Prefix bootstrap type to image reference + ref = b.Recipe.Header["bootstrap"] + ":" + ref + var imgCache *cache.Handle if !cp.b.Opts.NoCache { - // Grab the modified source ref from the cache - cp.srcRef, err = oci.ConvertReference(ctx, b.Opts.ImgCache, cp.srcRef, cp.topts) - if err != nil { - return fmt.Errorf("while converting reference: %w", err) - } + imgCache = cp.b.Opts.ImgCache } - // To to do the RootFS extraction we also have to have a location that - // contains *only* this image - cp.tmpfsRef, err = ocilayout.ParseReference(cp.b.TmpDir + ":" + "tmp") + // Fetch the image into a temporary containers/image oci layout dir. + cp.srcImg, err = ociimage.FetchToLayout(ctx, cp.topts, imgCache, ref, b.TmpDir) if err != nil { - return fmt.Errorf("while parsing reference: %w", err) - } - - err = cp.fetch(ctx) - if err != nil { - return fmt.Errorf("while fetching image: %w", err) + return err } - cp.imgConfig, err = cp.getConfig(ctx) + cf, err := cp.srcImg.ConfigFile() if err != nil { - return fmt.Errorf("while getting config: %w", err) + return err } + cp.imgConfig = cf.Config return nil } // Pack puts relevant objects in a Bundle. func (cp *OCIConveyorPacker) Pack(ctx context.Context) (*sytypes.Bundle, error) { - err := cp.unpackTmpfs(ctx) + err := cp.unpackRootfs(ctx) if err != nil { - return nil, fmt.Errorf("while unpacking tmpfs: %v", err) + return nil, fmt.Errorf("while unpacking rootfs: %v", err) } err = cp.insertBaseEnv() @@ -297,32 +223,6 @@ func (cp *OCIConveyorPacker) Pack(ctx context.Context) (*sytypes.Bundle, error) return cp.b, nil } -func (cp *OCIConveyorPacker) fetch(ctx context.Context) error { - // cp.srcRef contains the cache source reference - _, err := copy.Image(ctx, cp.policyCtx, cp.tmpfsRef, cp.srcRef, ©.Options{ - ReportWriter: io.Discard, - // nolint:staticcheck - SourceCtx: ocitransport.SystemContextFromTransportOptions(cp.topts), - RemoveSignatures: true, - }) - return err -} - -func (cp *OCIConveyorPacker) getConfig(ctx context.Context) (imgspecv1.ImageConfig, error) { - // nolint:staticcheck - img, err := cp.srcRef.NewImage(ctx, ocitransport.SystemContextFromTransportOptions(cp.topts)) - if err != nil { - return imgspecv1.ImageConfig{}, err - } - defer img.Close() - - imgSpec, err := img.OCIConfig(ctx) - if err != nil { - return imgspecv1.ImageConfig{}, err - } - return imgSpec.Config, nil -} - func (cp *OCIConveyorPacker) insertOCIConfig() error { conf, err := json.Marshal(cp.imgConfig) if err != nil { @@ -333,87 +233,28 @@ func (cp *OCIConveyorPacker) insertOCIConfig() error { return nil } -// Perform a dumb tar(gz) extraction with no chown, id remapping etc. -// This is needed for non-root handling of `oci-archive` as the extraction -// by containers/archive is failing when uid/gid don't match local machine -// and we're not root -func (cp *OCIConveyorPacker) extractArchive(src string, dst string) error { - f, err := os.Open(src) - if err != nil { +func (cp *OCIConveyorPacker) unpackRootfs(ctx context.Context) error { + if err := UnpackRootfs(ctx, cp.srcImg, cp.b.RootfsPath); err != nil { return err } - defer f.Close() - r := bufio.NewReader(f) - header, err := r.Peek(10) // read a few bytes without consuming - if err != nil { - return err + // If the `--fix-perms` flag was used, then modify the permissions so that + // content has owner rwX and we're done + if cp.b.Opts.FixPerms { + sylog.Warningf("The --fix-perms option modifies the filesystem permissions on the resulting container.") + sylog.Debugf("Modifying permissions for file/directory owners") + return FixPerms(cp.b.RootfsPath) } - gzipped := strings.Contains(http.DetectContentType(header), "x-gzip") - if gzipped { - r, err := gzip.NewReader(f) - if err != nil { - return err - } - defer r.Close() + // If `--fix-perms` was not used and this is a sandbox, scan for restrictive + // perms that would stop the user doing an `rm` without a chmod first, + // and warn if they exist + if cp.b.Opts.SandboxTarget { + sylog.Debugf("Scanning for restrictive permissions") + return CheckPerms(cp.b.RootfsPath) } - tr := tar.NewReader(r) - - for { - header, err := tr.Next() - - switch { - - // if no more files are found return - case err == io.EOF: - return nil - - // return any other error - case err != nil: - return err - - // if the header is nil, just skip it (not sure how this happens) - case header == nil: - continue - } - - // ZipSlip protection - don't escape from dst - // #nosec G305 - target := filepath.Join(dst, header.Name) - if !strings.HasPrefix(target, filepath.Clean(dst)+string(os.PathSeparator)) { - return fmt.Errorf("%s: illegal extraction path", target) - } - - // check the file type - switch header.Typeflag { - // if its a dir and it doesn't exist create it - case tar.TypeDir: - if _, err := os.Stat(target); err != nil { - if err := os.MkdirAll(target, 0o755); err != nil { - return err - } - } - // if it's a file create it - case tar.TypeReg: - f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) - if err != nil { - return err - } - defer f.Close() - - // copy over contents - if _, err := io.Copy(f, tr); err != nil { //nolint:gosec - return err - } - } - } -} - -func (cp *OCIConveyorPacker) unpackTmpfs(ctx context.Context) error { - // nolint:staticcheck - return unpackRootfs(ctx, cp.b, cp.tmpfsRef, ocitransport.SystemContextFromTransportOptions(cp.topts)) + return nil } func (cp *OCIConveyorPacker) insertBaseEnv() (err error) { diff --git a/internal/pkg/build/sources/oci_unpack.go b/internal/pkg/build/sources/oci_unpack.go index 19155aacd3..7ac358f476 100644 --- a/internal/pkg/build/sources/oci_unpack.go +++ b/internal/pkg/build/sources/oci_unpack.go @@ -15,25 +15,52 @@ package sources import ( "context" - "encoding/json" "errors" "fmt" "os" apexlog "github.com/apex/log" "github.com/apptainer/apptainer/internal/pkg/util/fs" - sytypes "github.com/apptainer/apptainer/pkg/build/types" "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/namespaces" - "github.com/containers/image/v5/types" - imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/opencontainers/umoci" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" umocilayer "github.com/opencontainers/umoci/oci/layer" "github.com/opencontainers/umoci/pkg/idtools" ) -// unpackRootfs extracts all of the layers of the given image reference into the rootfs of the provided bundle -func unpackRootfs(ctx context.Context, b *sytypes.Bundle, tmpfsRef types.ImageReference, sysCtx *types.SystemContext) (err error) { +// isExtractable checks if we have extractable layers in the image. Shouldn't be +// an ORAS artifact or similar. If we don't check, ggcr mutate.Extract will +// happily create an empty rootfs, leading to odd error messages elsewhere. +func isExtractable(img v1.Image) (bool, error) { + layers, err := img.Layers() + if err != nil { + return false, err + } + for _, l := range layers { + mt, err := l.MediaType() + if err != nil { + return false, err + } + if mt.IsLayer() { + return true, nil + } + } + return false, nil +} + +// UnpackRootfs extracts all of the layers of the given srcImage into destDir. +func UnpackRootfs(_ context.Context, srcImage v1.Image, destDir string) (err error) { + extractable, err := isExtractable(srcImage) + if err != nil { + return err + } + if !extractable { + return fmt.Errorf("no extractable OCI/Docker tar layers found in this image") + } + + flatTar := mutate.Extract(srcImage) + var mapOptions umocilayer.MapOptions loggerLevel := sylog.GetLevel() @@ -71,60 +98,59 @@ func unpackRootfs(ctx context.Context, b *sytypes.Bundle, tmpfsRef types.ImageRe mapOptions.GIDMappings = append(mapOptions.GIDMappings, gidMap) } - engineExt, err := umoci.OpenLayout(b.TmpDir) - if err != nil { - return fmt.Errorf("error opening layout: %s", err) - } - - // Obtain the manifest - imageSource, err := tmpfsRef.NewImageSource(ctx, sysCtx) - if err != nil { - return fmt.Errorf("error creating image source: %s", err) - } - manifestData, mediaType, err := imageSource.GetManifest(ctx, nil) - if err != nil { - return fmt.Errorf("error obtaining manifest source: %s", err) - } - if mediaType != imgspecv1.MediaTypeImageManifest { - return fmt.Errorf("error verifying manifest media type: %s", mediaType) - } - var manifest imgspecv1.Manifest - json.Unmarshal(manifestData, &manifest) - - // UnpackRootfs from umoci v0.4.2 expects a path to a non-existing directory - os.RemoveAll(b.RootfsPath) - // Unpack root filesystem unpackOptions := umocilayer.UnpackOptions{MapOptions: mapOptions} - err = umocilayer.UnpackRootfs(ctx, engineExt, b.RootfsPath, manifest, &unpackOptions) + err = umocilayer.UnpackLayer(destDir, flatTar, &unpackOptions) if err != nil { return fmt.Errorf("error unpacking rootfs: %s", err) } - // If the `--fix-perms` flag was used, then modify the permissions so that - // content has owner rwX and we're done - if b.Opts.FixPerms { - sylog.Warningf("The --fix-perms option modifies the filesystem permissions on the resulting container.") - sylog.Debugf("Modifying permissions for file/directory owners") - return sytypes.FixPerms(b.RootfsPath) - } + // No `--fix-perms` and no sandbox... we are fine + return err +} - // If `--fix-perms` was not used and this is a sandbox, scan for restrictive - // perms that would stop the user doing an `rm` without a chmod first, - // and warn if they exist - if b.Opts.SandboxTarget { - sylog.Debugf("Scanning for restrictive permissions") - return checkPerms(b.RootfsPath) - } +// FixPerms will work through the rootfs of this bundle, making sure that all +// files and directories have permissions set such that the owner can read, +// modify, delete. This brings us to the situation of <=3.4 +func FixPerms(rootfs string) (err error) { + errors := 0 + err = fs.PermWalk(rootfs, func(path string, f os.FileInfo, err error) error { + if err != nil { + sylog.Errorf("Unable to access rootfs path %s: %s", path, err) + errors++ + return nil + } - // No `--fix-perms` and no sandbox... we are fine + switch mode := f.Mode(); { + // Directories must have the owner 'rx' bits to allow traversal and reading on move, and the 'w' bit + // so their content can be deleted by the user when the rootfs/sandbox is deleted + case mode.IsDir(): + if err := os.Chmod(path, f.Mode().Perm()|0o700); err != nil { + sylog.Errorf("Error setting permission for %s: %s", path, err) + errors++ + } + case mode.IsRegular(): + // Regular files must have the owner 'r' bit so that everything can be read in order to + // copy or move the rootfs/sandbox around. Also, the `w` bit as the build does write into + // some files (e.g. resolv.conf) in the container rootfs. + if err := os.Chmod(path, f.Mode().Perm()|0o600); err != nil { + sylog.Errorf("Error setting permission for %s: %s", path, err) + errors++ + } + } + return nil + }) + + if errors > 0 { + err = fmt.Errorf("%d errors were encountered when setting permissions", errors) + } return err } -// checkPerms will work through the rootfs of this bundle, and find if any +// CheckPerms will work through the rootfs of this bundle, and find if any // directory does not have owner rwX - which may cause unexpected issues for a // user trying to look through, or delete a sandbox -func checkPerms(rootfs string) (err error) { +func CheckPerms(rootfs string) (err error) { // This is a locally defined error we can bubble up to cancel our recursive // structure. errRestrictivePerm := errors.New("restrictive file permission found") @@ -140,8 +166,7 @@ func checkPerms(rootfs string) (err error) { return fmt.Errorf("unable to access rootfs path %s: %s", path, err) } // Warn on any directory not `rwX` - technically other combinations may - // be traversable / removable... but are confusing to the user vs - // the Singularity 3.4 behavior. + // be traversable / removable if f.Mode().IsDir() && f.Mode().Perm()&0o700 != 0o700 { sylog.Debugf("Path %q has restrictive permissions", path) return errRestrictivePerm diff --git a/internal/pkg/client/oci/pull.go b/internal/pkg/client/oci/pull.go index 621dd17fa1..ee621e7fe5 100644 --- a/internal/pkg/client/oci/pull.go +++ b/internal/pkg/client/oci/pull.go @@ -20,7 +20,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/build" "github.com/apptainer/apptainer/internal/pkg/build/oci" "github.com/apptainer/apptainer/internal/pkg/cache" - "github.com/apptainer/apptainer/internal/pkg/ocitransport" + "github.com/apptainer/apptainer/internal/pkg/ociimage" "github.com/apptainer/apptainer/internal/pkg/util/fs" "github.com/apptainer/apptainer/internal/pkg/util/ociauth" buildtypes "github.com/apptainer/apptainer/pkg/build/types" @@ -41,8 +41,8 @@ type PullOptions struct { } // transportOptions maps PullOptions to OCI image transport options -func transportOptions(opts PullOptions) *ocitransport.TransportOptions { - return &ocitransport.TransportOptions{ +func transportOptions(opts PullOptions) *ociimage.TransportOptions { + return &ociimage.TransportOptions{ AuthConfig: opts.OciAuth, AuthFilePath: ociauth.ChooseAuthFile(opts.ReqAuthFile), Insecure: opts.NoHTTPS, diff --git a/internal/pkg/ociimage/fetch.go b/internal/pkg/ociimage/fetch.go new file mode 100644 index 0000000000..ac3f7acaf5 --- /dev/null +++ b/internal/pkg/ociimage/fetch.go @@ -0,0 +1,192 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package ociimage + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/pkg/sylog" + ggcrv1 "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// CachedImage will ensure that the provided v1.Image is present in the Apptainer +// OCI cache layout dir, and return a new v1.Image pointing to the cached copy. +func CachedImage(ctx context.Context, imgCache *cache.Handle, srcImg v1.Image) (v1.Image, error) { + if imgCache == nil || imgCache.IsDisabled() { + return nil, fmt.Errorf("undefined image cache") + } + + digest, err := srcImg.Digest() + if err != nil { + return nil, err + } + + layoutDir, err := imgCache.GetOciCacheDir(cache.OciBlobCacheType) + if err != nil { + return nil, err + } + + cachedRef := layoutDir + "@" + digest.String() + sylog.Debugf("Caching image to %s", cachedRef) + + OCISourceSink.WriteImage(srcImg, layoutDir) + + return OCISourceSink.Image(ctx, cachedRef, nil) +} + +// FetchToLayout will fetch the OCI image specified by imageRef to an OCI layout +// and return a v1.Image referencing it. If imgCache is non-nil, and enabled, +// the image will be fetched into Apptainer's cache - which is a multi-image +// OCI layout. If the cache is disabled, the image will be fetched into a +// subdirectory of the provided tmpDir. The caller is responsible for cleaning +// up tmpDir. +func FetchToLayout(ctx context.Context, tOpts *TransportOptions, imgCache *cache.Handle, imageURI, tmpDir string) (ggcrv1.Image, error) { + // oci-archive - Perform a tar extraction first, and handle as an oci layout. + if strings.HasPrefix(imageURI, "oci-archive:") { + var tmpDir string + tmpDir, err := os.MkdirTemp(tOpts.TmpDir, "temp-oci-") + if err != nil { + return nil, fmt.Errorf("could not create temporary oci directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + // oci-archive:[:tag] + refParts := strings.SplitN(imageURI, ":", 3) + sylog.Debugf("Extracting oci-archive %q to %q", refParts[1], tmpDir) + err = extractArchive(refParts[1], tmpDir) + if err != nil { + return nil, fmt.Errorf("error extracting the OCI archive file: %v", err) + } + // We may or may not have had a ':tag' in the source to handle + imageURI = "oci:" + tmpDir + if len(refParts) == 3 { + imageURI = imageURI + ":" + refParts[2] + } + } + + srcType, srcRef, err := URItoSourceSinkRef(imageURI) + if err != nil { + return nil, err + } + + srcImg, err := srcType.Image(ctx, srcRef, tOpts) + if err != nil { + return nil, err + } + + if imgCache != nil && !imgCache.IsDisabled() { + // Ensure the image is cached, and return reference to the cached image. + return CachedImage(ctx, imgCache, srcImg) + } + + // No cache - write to layout directory provided + tmpLayout, err := os.MkdirTemp(tmpDir, "layout-") + if err != nil { + return nil, err + } + sylog.Debugf("Copying %q to temporary layout at %q", srcRef, tmpLayout) + if err = OCISourceSink.WriteImage(srcImg, tmpLayout); err != nil { + return nil, err + } + + return OCISourceSink.Image(ctx, tmpLayout, tOpts) +} + +// Perform a dumb tar(gz) extraction with no chown, id remapping etc. +// This is needed for non-root handling of `oci-archive` as the extraction +// by containers/archive is failing when uid/gid don't match local machine +// and we're not root +func extractArchive(src string, dst string) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + r := bufio.NewReader(f) + header, err := r.Peek(10) // read a few bytes without consuming + if err != nil { + return err + } + gzipped := strings.Contains(http.DetectContentType(header), "x-gzip") + + if gzipped { + r, err := gzip.NewReader(f) + if err != nil { + return err + } + defer r.Close() + } + + tr := tar.NewReader(r) + + for { + header, err := tr.Next() + switch { + // if no more files are found return + case err == io.EOF: + return nil + + // return any other error + case err != nil: + return err + + // if the header is nil, just skip it (not sure how this happens) + case header == nil: + continue + } + + // ZipSlip protection - don't escape from dst + //#nosec G305 + target := filepath.Join(dst, header.Name) + if !strings.HasPrefix(target, filepath.Clean(dst)+string(os.PathSeparator)) { + return fmt.Errorf("%s: illegal extraction path", target) + } + + // check the file type + switch header.Typeflag { + // if its a dir and it doesn't exist create it + case tar.TypeDir: + if _, err := os.Stat(target); err != nil { + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + } + // if it's a file create it + case tar.TypeReg: + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + defer f.Close() + + // copy over contents + for { + if _, err := io.CopyN(f, tr, 1024); err != nil { + if err == io.EOF { + break + } + return err + } + } + } + } +} diff --git a/internal/pkg/ociimage/sourcesink.go b/internal/pkg/ociimage/sourcesink.go new file mode 100644 index 0000000000..e751b2e726 --- /dev/null +++ b/internal/pkg/ociimage/sourcesink.go @@ -0,0 +1,181 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package ociimage + +import ( + "context" + "fmt" + "strings" + + "github.com/apptainer/apptainer/internal/pkg/util/ociauth" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/docker/docker/client" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/daemon" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +type SourceSink int + +const ( + UnknownSourceSink SourceSink = iota + RegistrySourceSink + OCISourceSink + TarballSourceSink + DaemonSourceSink +) + +func getDockerImage(ctx context.Context, src string, tOpts *TransportOptions) (v1.Image, error) { + var nameOpts []name.Option + if tOpts != nil && tOpts.Insecure { + nameOpts = append(nameOpts, name.Insecure) + } + + srcRef, err := name.ParseReference(src, nameOpts...) + if err != nil { + return nil, err + } + + pullOpts := []remote.Option{ + remote.WithContext(ctx), + } + + if tOpts != nil { + pullOpts = append(pullOpts, + remote.WithPlatform(tOpts.Platform), + ociauth.AuthOptn(tOpts.AuthConfig, tOpts.AuthFilePath)) + } + + return remote.Image(srcRef, pullOpts...) +} + +// getOCIImage retrieves an image from a layout ref provided in [@digest] format. +// If no digest is provided, and there is only one image in the layout, it will be returned. +// A digest must be specified when retrieving an image from a layout containing multiple images. +func getOCIImage(src string) (v1.Image, error) { + refParts := strings.SplitN(src, "@", 2) + + lp, err := layout.FromPath(refParts[0]) + if err != nil { + return nil, err + } + + ii, err := lp.ImageIndex() + if err != nil { + return nil, err + } + + im, err := ii.IndexManifest() + if err != nil { + return nil, err + } + + if len(im.Manifests) < 1 { + return nil, fmt.Errorf("no images found in layout %s", src) + } + + if len(refParts) < 2 && len(im.Manifests) != 1 { + return nil, fmt.Errorf("must specify a digest - layout contains multiple images") + } + if len(refParts) == 1 { + return lp.Image(im.Manifests[0].Digest) + } + + for _, mf := range im.Manifests { + sylog.Debugf("%v =? %v", mf.Digest.String(), refParts[1]) + if mf.Digest.String() == refParts[1] { + return ii.Image(mf.Digest) + } + } + + return nil, fmt.Errorf("image %q not found in layout", src) +} + +func getDaemonImage(ctx context.Context, src string, tOpts *TransportOptions) (v1.Image, error) { + var nameOpts []name.Option + if tOpts != nil && tOpts.Insecure { + nameOpts = append(nameOpts, name.Insecure) + } + + srcRef, err := name.ParseReference(src, nameOpts...) + if err != nil { + return nil, err + } + + dOpts := []daemon.Option{ + daemon.WithContext(ctx), + } + + if tOpts != nil && tOpts.DockerDaemonHost != "" { + dc, err := client.NewClientWithOpts(client.WithHost(tOpts.DockerDaemonHost)) + if err != nil { + return nil, err + } + dOpts = append(dOpts, daemon.WithClient(dc)) + } + + return daemon.Image(srcRef, dOpts...) +} + +func (ss SourceSink) Reference(s string, tOpts *TransportOptions) (name.Reference, bool) { + switch ss { + case RegistrySourceSink, DaemonSourceSink: + var nameOpts []name.Option + if tOpts != nil && tOpts.Insecure { + nameOpts = append(nameOpts, name.Insecure) + } + srcRef, err := name.ParseReference(s, nameOpts...) + if err != nil { + return nil, false + } + return srcRef, true + default: + return nil, false + } +} + +func (ss SourceSink) Image(ctx context.Context, ref string, tOpts *TransportOptions) (v1.Image, error) { + switch ss { + case RegistrySourceSink: + return getDockerImage(ctx, ref, tOpts) + case TarballSourceSink: + return tarball.ImageFromPath(ref, nil) + case OCISourceSink: + return getOCIImage(ref) + case DaemonSourceSink: + return getDaemonImage(ctx, ref, tOpts) + case UnknownSourceSink: + return nil, errUnsupportedTransport + default: + return nil, errUnsupportedTransport + } +} + +func (ss SourceSink) WriteImage(img v1.Image, dstName string) error { + switch ss { + case OCISourceSink: + lp, err := layout.FromPath(dstName) + if err != nil { + lp, err = layout.Write(dstName, empty.Index) + if err != nil { + return err + } + } + return lp.AppendImage(img) + case UnknownSourceSink: + return errUnsupportedTransport + default: + return errUnsupportedTransport + } +} diff --git a/internal/pkg/ocitransport/ocitransport.go b/internal/pkg/ociimage/transport.go similarity index 86% rename from internal/pkg/ocitransport/ocitransport.go rename to internal/pkg/ociimage/transport.go index ae9eef1275..0a88b0f82a 100644 --- a/internal/pkg/ocitransport/ocitransport.go +++ b/internal/pkg/ociimage/transport.go @@ -7,9 +7,10 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package ocitransport +package ociimage import ( + "errors" "fmt" "os" "runtime" @@ -29,6 +30,8 @@ import ( var ociTransports = []string{"docker", "docker-archive", "docker-daemon", "oci", "oci-archive"} +var errUnsupportedTransport = errors.New("unsupported transport") + // SupportedTransport returns whether or not the transport given is supported. To fit within a switch/case // statement, this function will return transport if it is supported func SupportedTransport(transport string) string { @@ -163,14 +166,14 @@ func SystemContextFromTransportOptions(tOpts *TransportOptions) *types.SystemCon return &sc } -// defaultPolicy is Apptainer's default containers/image OCI signature verification policy - accept anything. +// DefaultPolicy is Apptainer's default containers/image OCI signature verification policy - accept anything. func DefaultPolicy() (*signature.PolicyContext, error) { policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} return signature.NewPolicyContext(policy) } -// parseImageRef parses a uri-like OCI image reference into a containers/image types.ImageReference. -func ParseImageRef(imageRef string) (types.ImageReference, error) { +// URIToImageReference parses a uri-like OCI image reference into a containers/image types.ImageReference. +func URIToImageReference(imageRef string) (types.ImageReference, error) { parts := strings.SplitN(imageRef, ":", 2) if len(parts) < 2 { return nil, fmt.Errorf("could not parse image ref: %s", imageRef) @@ -191,15 +194,38 @@ func ParseImageRef(imageRef string) (types.ImageReference, error) { case "oci-archive": srcRef, err = ociarchive.ParseReference(parts[1]) default: - return nil, fmt.Errorf("cannot create an OCI container from %s source", parts[0]) + return nil, errUnsupportedTransport } if err != nil { - return nil, fmt.Errorf("invalid image source: %v", err) + return nil, err } return srcRef, nil } +// URItoSourceSinkRef parses a uri-like OCI image reference into a SourceSink and ref +func URItoSourceSinkRef(imageURI string) (SourceSink, string, error) { + parts := strings.SplitN(imageURI, ":", 2) + if len(parts) < 2 { + return UnknownSourceSink, "", fmt.Errorf("could not parse image ref: %s", imageURI) + } + + switch parts[0] { + case "docker": + // Remove slashes from docker:// URI + parts[1] = strings.TrimPrefix(parts[1], "//") + return RegistrySourceSink, parts[1], nil + case "docker-archive": + return TarballSourceSink, parts[1], nil + case "docker-daemon": + return DaemonSourceSink, parts[1], nil + case "oci": + return OCISourceSink, parts[1], nil + } + + return UnknownSourceSink, "", errUnsupportedTransport +} + func defaultSysCtx() *types.SystemContext { sysCtx := &types.SystemContext{ OSChoice: "linux", diff --git a/internal/pkg/ocitransport/ocitransport_test.go b/internal/pkg/ociimage/transport_test.go similarity index 98% rename from internal/pkg/ocitransport/ocitransport_test.go rename to internal/pkg/ociimage/transport_test.go index a18cb2f205..efa849a808 100644 --- a/internal/pkg/ocitransport/ocitransport_test.go +++ b/internal/pkg/ociimage/transport_test.go @@ -7,7 +7,7 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package ocitransport +package ociimage import ( "testing" diff --git a/internal/pkg/ociplatform/cpuinfo.go b/internal/pkg/ociplatform/cpuinfo.go new file mode 100644 index 0000000000..37ad463763 --- /dev/null +++ b/internal/pkg/ociplatform/cpuinfo.go @@ -0,0 +1,47 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ociplatform + +import ( + "runtime" + "sync" + + "github.com/apptainer/apptainer/pkg/sylog" +) + +// Present the ARM instruction set architecture, eg: v7, v8 +// Don't use this value directly; call cpuVariant() instead. +var cpuVariantValue string + +var cpuVariantOnce sync.Once + +func CPUVariant() string { + cpuVariantOnce.Do(func() { + if isArmArch(runtime.GOARCH) { + var err error + cpuVariantValue, err = getCPUVariant() + if err != nil { + sylog.Errorf("Error getCPUVariant for OS %s: %v", runtime.GOOS, err) + } + } + }) + return cpuVariantValue +} diff --git a/internal/pkg/ociplatform/cpuinfo_linux.go b/internal/pkg/ociplatform/cpuinfo_linux.go new file mode 100644 index 0000000000..b3ffee540b --- /dev/null +++ b/internal/pkg/ociplatform/cpuinfo_linux.go @@ -0,0 +1,167 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ociplatform + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "os" + "runtime" + "strings" + + "golang.org/x/sys/unix" +) + +var ( + errNotFound = errors.New("not found") + errInvalidArgument = errors.New("invalid argument") +) + +// getMachineArch retrieves the machine architecture through system call +func getMachineArch() (string, error) { + var uname unix.Utsname + err := unix.Uname(&uname) + if err != nil { + return "", err + } + + arch := string(uname.Machine[:bytes.IndexByte(uname.Machine[:], 0)]) + + return arch, nil +} + +// For Linux, the kernel has already detected the ABI, ISA and Features. +// So we don't need to access the ARM registers to detect platform information +// by ourselves. We can just parse these information from /proc/cpuinfo +func getCPUInfo(pattern string) (info string, err error) { + cpuinfo, err := os.Open("/proc/cpuinfo") + if err != nil { + return "", err + } + defer cpuinfo.Close() + + // Start to Parse the Cpuinfo line by line. For SMP SoC, we parse + // the first core is enough. + scanner := bufio.NewScanner(cpuinfo) + for scanner.Scan() { + newline := scanner.Text() + list := strings.Split(newline, ":") + + if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) { + return strings.TrimSpace(list[1]), nil + } + } + + // Check whether the scanner encountered errors + err = scanner.Err() + if err != nil { + return "", err + } + + return "", fmt.Errorf("getCPUInfo for pattern %s: %w", pattern, errNotFound) +} + +// getCPUVariantFromArch get CPU variant from arch through a system call +func getCPUVariantFromArch(arch string) (string, error) { + var variant string + + arch = strings.ToLower(arch) + + if arch == "aarch64" { + variant = "8" + } else if arch[0:4] == "armv" && len(arch) >= 5 { + // Valid arch format is in form of armvXx + switch arch[3:5] { + case "v8": + variant = "8" + case "v7": + variant = "7" + case "v6": + variant = "6" + case "v5": + variant = "5" + case "v4": + variant = "4" + case "v3": + variant = "3" + default: + variant = "unknown" + } + } else { + return "", fmt.Errorf("getCPUVariantFromArch invalid arch: %s, %w", arch, errInvalidArgument) + } + return variant, nil +} + +// getCPUVariant returns cpu variant for ARM +// We first try reading "Cpu architecture" field from /proc/cpuinfo +// If we can't find it, then fall back using a system call +// This is to cover running ARM in emulated environment on x86 host as this field in /proc/cpuinfo +// was not present. +func getCPUVariant() (string, error) { + variant, err := getCPUInfo("Cpu architecture") + if err != nil { + if errors.Is(err, errNotFound) { + // Let's try getting CPU variant from machine architecture + arch, err := getMachineArch() + if err != nil { + return "", fmt.Errorf("failure getting machine architecture: %v", err) + } + + variant, err = getCPUVariantFromArch(arch) + if err != nil { + return "", fmt.Errorf("failure getting CPU variant from machine architecture: %v", err) + } + } else { + return "", fmt.Errorf("failure getting CPU variant: %v", err) + } + } + + // handle edge case for Raspberry Pi ARMv6 devices (which due to a kernel quirk, report "CPU architecture: 7") + // https://www.raspberrypi.org/forums/viewtopic.php?t=12614 + if runtime.GOARCH == "arm" && variant == "7" { + model, err := getCPUInfo("model name") + if err == nil && strings.HasPrefix(strings.ToLower(model), "armv6-compatible") { + variant = "6" + } + } + + switch strings.ToLower(variant) { + case "8", "aarch64": + variant = "v8" + case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)": + variant = "v7" + case "6", "6tej": + variant = "v6" + case "5", "5t", "5te", "5tej": + variant = "v5" + case "4", "4t": + variant = "v4" + case "3": + variant = "v3" + default: + variant = "unknown" + } + + return variant, nil +} diff --git a/internal/pkg/ociplatform/cpuinfo_linux_test.go b/internal/pkg/ociplatform/cpuinfo_linux_test.go new file mode 100644 index 0000000000..fe7050cd44 --- /dev/null +++ b/internal/pkg/ociplatform/cpuinfo_linux_test.go @@ -0,0 +1,140 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ociplatform + +import ( + "errors" + "runtime" + "testing" +) + +func TestCPUVariant(t *testing.T) { + if !isArmArch(runtime.GOARCH) { + t.Skip("only relevant on linux/arm") + } + + variants := []string{"v8", "v7", "v6", "v5", "v4", "v3"} + + p, err := getCPUVariant() + if err != nil { + t.Fatalf("Error getting CPU variant: %v", err) + return + } + + for _, variant := range variants { + if p == variant { + t.Logf("got valid variant as expected: %#v = %#v", p, variant) + return + } + } + + t.Fatalf("could not get valid variant as expected: %v", variants) +} + +func TestGetCPUVariantFromArch(t *testing.T) { + for _, testcase := range []struct { + name string + input string + output string + expectedErr error + }{ + { + name: "Test aarch64", + input: "aarch64", + output: "8", + expectedErr: nil, + }, + { + name: "Test Armv8 with capital", + input: "Armv8", + output: "8", + expectedErr: nil, + }, + { + name: "Test armv7", + input: "armv7", + output: "7", + expectedErr: nil, + }, + { + name: "Test armv6", + input: "armv6", + output: "6", + expectedErr: nil, + }, + { + name: "Test armv5", + input: "armv5", + output: "5", + expectedErr: nil, + }, + { + name: "Test armv4", + input: "armv4", + output: "4", + expectedErr: nil, + }, + { + name: "Test armv3", + input: "armv3", + output: "3", + expectedErr: nil, + }, + { + name: "Test unknown input", + input: "armv9", + output: "unknown", + expectedErr: nil, + }, + { + name: "Test invalid input which doesn't start with armv", + input: "armxxxx", + output: "", + expectedErr: errInvalidArgument, + }, + { + name: "Test invalid input whose length is less than 5", + input: "armv", + output: "", + expectedErr: errInvalidArgument, + }, + } { + t.Run(testcase.name, func(t *testing.T) { + t.Logf("input: %v", testcase.input) + + variant, err := getCPUVariantFromArch(testcase.input) + + if err == nil { + if testcase.expectedErr != nil { + t.Fatalf("Expect to get error: %v, however no error got", testcase.expectedErr) + } else { + if variant != testcase.output { + t.Fatalf("Expect to get variant: %v, however %v returned", testcase.output, variant) + } + } + } else { + if !errors.Is(err, testcase.expectedErr) { + t.Fatalf("Expect to get error: %v, however error %v returned", testcase.expectedErr, err) + } + } + }) + } +} diff --git a/internal/pkg/ociplatform/database.go b/internal/pkg/ociplatform/database.go new file mode 100644 index 0000000000..5484ca9b21 --- /dev/null +++ b/internal/pkg/ociplatform/database.go @@ -0,0 +1,70 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ociplatform + +import ( + "strings" +) + +// isArmArch returns true if the architecture is ARM. +// +// The arch value should be normalized before being passed to this function. +func isArmArch(arch string) bool { + switch arch { + case "arm", "arm64": + return true + } + return false +} + +// normalizeArch normalizes the architecture. +func normalizeArch(arch, variant string) (string, string) { + arch, variant = strings.ToLower(arch), strings.ToLower(variant) + switch arch { + case "i386": + arch = "386" + variant = "" + case "x86_64", "x86-64": + arch = "amd64" + variant = "" + case "aarch64", "arm64": + arch = "arm64" + switch variant { + case "8", "v8": + variant = "" + } + case "armhf": + arch = "arm" + variant = "v7" + case "armel": + arch = "arm" + variant = "v6" + case "arm": + switch variant { + case "", "7": + variant = "v7" + case "5", "6", "8": + variant = "v" + variant + } + } + + return arch, variant +} diff --git a/internal/pkg/ociplatform/ociplatform.go b/internal/pkg/ociplatform/ociplatform.go new file mode 100644 index 0000000000..3fdb302755 --- /dev/null +++ b/internal/pkg/ociplatform/ociplatform.go @@ -0,0 +1,84 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package ociplatform + +import ( + "fmt" + "runtime" + + "github.com/apptainer/apptainer/pkg/sylog" + ggcrv1 "github.com/google/go-containerregistry/pkg/v1" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// CheckImagePlatform ensures that an image reference satisfies the provided platform requirements. +func CheckImagePlatform(platform v1.Platform, img v1.Image) error { + cf, err := img.ConfigFile() + if err != nil { + return err + } + + if cf.Platform() == nil { + sylog.Warningf("OCI image doesn't declare a platform. It may not be compatible with this system.") + return nil + } + + if cf.Platform().Satisfies(platform) { + return nil + } + + return fmt.Errorf("image (%s) does not satisfy required platform (%s)", cf.Platform(), platform) +} + +func DefaultPlatform() (*ggcrv1.Platform, error) { + os := runtime.GOOS + arch := runtime.GOARCH + variant := CPUVariant() + + if os != "linux" { + return nil, fmt.Errorf("%q is not a valid platform OS for apptainer", runtime.GOOS) + } + + arch, variant = normalizeArch(arch, variant) + + return &ggcrv1.Platform{ + OS: os, + Architecture: arch, + Variant: variant, + }, nil +} + +func PlatformFromString(p string) (*ggcrv1.Platform, error) { + plat, err := ggcrv1.ParsePlatform(p) + if err != nil { + return nil, err + } + if plat.OS != "linux" { + return nil, fmt.Errorf("%q is not a valid platform OS for apptainer", plat.OS) + } + + plat.Architecture, plat.Variant = normalizeArch(plat.Architecture, plat.Variant) + + return plat, nil +} + +func PlatformFromArch(a string) (*ggcrv1.Platform, error) { + if runtime.GOOS != "linux" { + return nil, fmt.Errorf("%q is not a valid platform OS for apptainer", runtime.GOOS) + } + + arch, variant := normalizeArch(a, "") + + return &ggcrv1.Platform{ + OS: runtime.GOOS, + Architecture: arch, + Variant: variant, + }, nil +} diff --git a/internal/pkg/ociplatform/ociplatform_test.go b/internal/pkg/ociplatform/ociplatform_test.go new file mode 100644 index 0000000000..3319afc94b --- /dev/null +++ b/internal/pkg/ociplatform/ociplatform_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package ociplatform + +import ( + "reflect" + "testing" + + ggcrv1 "github.com/google/go-containerregistry/pkg/v1" +) + +func TestPlatformFromString(t *testing.T) { + tests := []struct { + name string + plat string + want *ggcrv1.Platform + wantErr bool + }{ + { + name: "BadString", + plat: "os/arch/variant/extra", + want: nil, + wantErr: true, + }, + { + name: "UnsupportedWindows", + plat: "windows/amd64", + want: nil, + wantErr: true, + }, + { + name: "GoodAMD64", + plat: "linux/amd64", + want: &ggcrv1.Platform{OS: "linux", Architecture: "amd64", Variant: ""}, + wantErr: false, + }, + { + name: "NormalizeARM", + plat: "linux/arm", + want: &ggcrv1.Platform{OS: "linux", Architecture: "arm", Variant: "v7"}, + wantErr: false, + }, + { + name: "NormalizeARM64/v8", + plat: "linux/arm64/v8", + want: &ggcrv1.Platform{OS: "linux", Architecture: "arm64", Variant: ""}, + wantErr: false, + }, + { + name: "NormalizeAARCH64", + plat: "linux/aarch64", + want: &ggcrv1.Platform{OS: "linux", Architecture: "arm64", Variant: ""}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := PlatformFromString(tt.plat) + if (err != nil) != tt.wantErr { + t.Errorf("PlatformFromString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("PlatformFromString() = %v, want %v", got, tt.want) + } + }) + } +}