Add configuration options to local transfer service

Signed-off-by: Tony Fang <nhfang@amazon.com>
containerd · Feb 25, 2023 · 4730539 · 4730539
1 parent e366fac
commit 4730539
Show file tree

Hide file tree

Showing 14 changed files with 209 additions and 82 deletions.
diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go
@@ -22,6 +22,9 @@ import (
 	"os"
 	"time"
 
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/urfave/cli"
+
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cmd/ctr/commands"
 	"github.com/containerd/containerd/images/archive"
@@ -30,7 +33,6 @@ import (
 	tarchive "github.com/containerd/containerd/pkg/transfer/archive"
 	"github.com/containerd/containerd/pkg/transfer/image"
 	"github.com/containerd/containerd/platforms"
-	"github.com/urfave/cli"
 )
 
 var importCommand = cli.Command{
@@ -127,9 +129,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
 				opts = append(opts, image.WithNamedPrefix(prefix, overwrite))
 			}
 
-			// TODO: Add platform options
+			var platSpec ocispec.Platform
+			//Only when all-platforms not specified, we will check platform value
+			//Implicitly if the platforms is empty, it means all-platforms
+			if !context.Bool("all-platforms") {
+				//If platform specified, use that one, if not use default
+				if platform := context.String("platform"); platform != "" {
+					platSpec, err = platforms.Parse(platform)
+					if err != nil {
+						return err
+					}
+				} else {
+					platSpec = platforms.DefaultSpec()
+				}
+				opts = append(opts, image.WithPlatforms(platSpec))
+			}
 
-			// TODO: Add unpack options
+			if !context.Bool("no-unpack") {
+				snapshotter := context.String("snapshotter")
+				//If OS field is not empty, it means platSpec was updated in the above block
+				//i.e all-platforms was not specified
+				if platSpec.OS != "" {
+					opts = append(opts, image.WithUnpack(platSpec, snapshotter))
+				} else {
+					//empty spec means all platforms
+					var emptySpec ocispec.Platform
+					opts = append(opts, image.WithUnpack(emptySpec, snapshotter))
+				}
+			}
 
 			is := image.NewStore(context.String("index-name"), opts...)
 

diff --git a/cmd/ctr/commands/images/pull.go b/cmd/ctr/commands/images/pull.go
@@ -32,6 +32,7 @@ import (
 	"github.com/containerd/containerd/pkg/progress"
 	"github.com/containerd/containerd/pkg/transfer"
 	"github.com/containerd/containerd/pkg/transfer/image"
+	"github.com/containerd/containerd/pkg/transfer/registry"
 	"github.com/containerd/containerd/platforms"
 	"github.com/opencontainers/image-spec/identity"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -112,6 +113,11 @@ command. As part of this process, we do the following:
 					p = append(p, platforms.DefaultSpec())
 				}
 				sopts = append(sopts, image.WithPlatforms(p...))
+
+				//set unpack configuration
+				for _, platform := range p {
+					sopts = append(sopts, image.WithUnpack(platform, context.String("snapshotter")))
+				}
 			}
 			// TODO: Support unpack for all platforms..?
 			// Pass in a *?
@@ -125,7 +131,7 @@ command. As part of this process, we do the following:
 				sopts = append(sopts, image.WithAllMetadata)
 			}
 
-			reg := image.NewOCIRegistry(ref, nil, ch)
+			reg := registry.NewOCIRegistry(ref, nil, ch)
 			is := image.NewStore(ref, sopts...)
 
 			pf, done := ProgressHandler(ctx, os.Stdout)

diff --git a/cmd/ctr/commands/images/push.go b/cmd/ctr/commands/images/push.go
@@ -34,6 +34,7 @@ import (
 	"github.com/containerd/containerd/pkg/progress"
 	"github.com/containerd/containerd/pkg/transfer"
 	"github.com/containerd/containerd/pkg/transfer/image"
+	"github.com/containerd/containerd/pkg/transfer/registry"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
@@ -103,7 +104,7 @@ var pushCommand = cli.Command{
 			if local == "" {
 				local = ref
 			}
-			reg := image.NewOCIRegistry(ref, nil, ch)
+			reg := registry.NewOCIRegistry(ref, nil, ch)
 			is := image.NewStore(local)
 
 			pf, done := ProgressHandler(ctx, os.Stdout)

diff --git a/cmd/ctr/commands/resolver.go b/cmd/ctr/commands/resolver.go
@@ -32,7 +32,7 @@ import (
 
 	"github.com/containerd/console"
 	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/pkg/transfer/image"
+	"github.com/containerd/containerd/pkg/transfer/registry"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/remotes/docker/config"
@@ -218,7 +218,7 @@ type staticCredentials struct {
 }
 
 // NewStaticCredentials gets credentials from passing in cli context
-func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (image.CredentialHelper, error) {
+func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref string) (registry.CredentialHelper, error) {
 	username := clicontext.String("user")
 	var secret string
 	if i := strings.IndexByte(username, ':'); i > 0 {
@@ -248,12 +248,12 @@ func NewStaticCredentials(ctx gocontext.Context, clicontext *cli.Context, ref st
 	}, nil
 }
 
-func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (image.Credentials, error) {
+func (sc *staticCredentials) GetCredentials(ctx gocontext.Context, ref, host string) (registry.Credentials, error) {
 	if ref == sc.ref {
-		return image.Credentials{
+		return registry.Credentials{
 			Username: sc.username,
 			Secret:   sc.secret,
 		}, nil
 	}
-	return image.Credentials{}, nil
+	return registry.Credentials{}, nil
 }
diff --git a/pkg/transfer/image/imagestore.go b/pkg/transfer/image/imagestore.go
@@ -27,8 +27,8 @@ import (
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/images/archive"
 	"github.com/containerd/containerd/pkg/streaming"
+	"github.com/containerd/containerd/pkg/transfer"
 	"github.com/containerd/containerd/pkg/transfer/plugins"
-	"github.com/containerd/containerd/pkg/unpack"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/typeurl/v2"
@@ -51,7 +51,7 @@ type Store struct {
 	// extraReferences are used to store or lookup multiple references
 	extraReferences []Reference
 
-	unpacks []UnpackConfiguration
+	unpacks []transfer.UnpackConfiguration
 }
 
 // Reference is used to create or find a reference for an image
@@ -84,14 +84,6 @@ type Reference struct {
 	SkipNamedDigest bool
 }
 
-// UnpackConfiguration specifies the platform and snapshotter to use for resolving
-// the unpack Platform, if snapshotter is not specified the platform default will
-// be used.
-type UnpackConfiguration struct {
-	Platform    ocispec.Platform
-	Snapshotter string
-}
-
 // StoreOpt defines options when configuring an image store source or destination
 type StoreOpt func(*Store)
 
@@ -171,7 +163,7 @@ func WithExtraReference(name string) StoreOpt {
 // WithUnpack specifies a platform to unpack for and an optional snapshotter to use
 func WithUnpack(p ocispec.Platform, snapshotter string) StoreOpt {
 	return func(s *Store) {
-		s.unpacks = append(s.unpacks, UnpackConfiguration{
+		s.unpacks = append(s.unpacks, transfer.UnpackConfiguration{
 			Platform:    p,
 			Snapshotter: snapshotter,
 		})
@@ -333,11 +325,11 @@ func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, err
 	return store.Get(ctx, is.imageName)
 }
 
-func (is *Store) UnpackPlatforms() []unpack.Platform {
-	unpacks := make([]unpack.Platform, len(is.unpacks))
+func (is *Store) UnpackPlatforms() []transfer.UnpackConfiguration {
+	unpacks := make([]transfer.UnpackConfiguration, len(is.unpacks))
 	for i, uc := range is.unpacks {
-		unpacks[i].SnapshotterKey = uc.Snapshotter
-		unpacks[i].Platform = platforms.Only(uc.Platform)
+		unpacks[i].Snapshotter = uc.Snapshotter
+		unpacks[i].Platform = uc.Platform
 	}
 	return unpacks
 }
@@ -424,7 +416,7 @@ func referencesFromProto(references []*transfertypes.ImageReference) []Reference
 	}
 	return or
 }
-func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguration {
+func unpackToProto(uc []transfer.UnpackConfiguration) []*transfertypes.UnpackConfiguration {
 	auc := make([]*transfertypes.UnpackConfiguration, len(uc))
 	for i := range uc {
 		p := types.Platform{
@@ -440,8 +432,8 @@ func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguratio
 	return auc
 }
 
-func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []UnpackConfiguration {
-	uc := make([]UnpackConfiguration, len(auc))
+func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []transfer.UnpackConfiguration {
+	uc := make([]transfer.UnpackConfiguration, len(auc))
 	for i := range auc {
 		uc[i].Snapshotter = auc[i].Snapshotter
 		if auc[i].Platform != nil {

diff --git a/pkg/transfer/local/import.go b/pkg/transfer/local/import.go
@@ -19,13 +19,16 @@ package local
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/pkg/transfer"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/containerd/containerd/pkg/unpack"
 )
 
 func (ts *localTransferService) importStream(ctx context.Context, i transfer.ImageImporter, is transfer.ImageStorer, tops *transfer.Config) error {
@@ -46,12 +49,16 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
 		return err
 	}
 
-	var descriptors []ocispec.Descriptor
+	var (
+		descriptors []ocispec.Descriptor
+		handler     images.Handler
+		unpacker    *unpack.Unpacker
+	)
 
 	// If save index, add index
 	descriptors = append(descriptors, index)
 
-	var handler images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+	var handlerFunc images.HandlerFunc = func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		// Only save images at top level
 		if desc.Digest != index.Digest {
 			return images.Children(ctx, ts.content, desc)
@@ -76,7 +83,33 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima
 	}
 
 	if f, ok := is.(transfer.ImageFilterer); ok {
-		handler = f.ImageFilter(handler, ts.content)
+		handlerFunc = f.ImageFilter(handlerFunc, ts.content)
+	}
+
+	handler = images.Handlers(handlerFunc)
+
+	// First find suitable platforms to unpack into
+	//If image storer is also an unpacker type, i.e implemented UnpackPlatforms() func
+	if iu, ok := is.(transfer.ImageUnpacker); ok {
+		unpacks := iu.UnpackPlatforms()
+		if len(unpacks) > 0 {
+			uopts := []unpack.UnpackerOpt{}
+			for _, u := range unpacks {
+				matched, mu := getSupportedPlatform(u, ts.config.UnpackPlatforms)
+				if matched {
+					uopts = append(uopts, unpack.WithUnpackPlatform(mu))
+				}
+			}
+
+			if ts.config.DuplicationSuppressor != nil {
+				uopts = append(uopts, unpack.WithDuplicationSuppressor(ts.config.DuplicationSuppressor))
+			}
+			unpacker, err = unpack.NewUnpacker(ctx, ts.content, uopts...)
+			if err != nil {
+				return fmt.Errorf("unable to initialize unpacker: %w", err)
+			}
+			handler = unpacker.Unpack(handler)
+		}
 	}
 
 	if err := images.WalkNotEmpty(ctx, handler, index); err != nil {