From fed26d5b3c9fa45afcf818f29ff30db8a16dd14c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 15 Jul 2023 20:42:29 +0200 Subject: [PATCH 001/105] vendor: github.com/moby/buildkit v0.11.7-dev full diff: https://github.com/moby/buildkit/compare/0a15675913b7...616c3f613b54a893df758428c51fad63ae2ccb7d Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 0ec73a789298d9102e71e29d4128cf02cb80b62e) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/buildkit.yml | 3 +-- builder/builder-next/worker/worker.go | 2 +- vendor.mod | 2 +- vendor.sum | 4 ++-- vendor/modules.txt | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 3e065c7a00636..7de6e0c22f941 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -13,8 +13,7 @@ on: pull_request: env: - # FIXME(thaJeztah): update to newer go versions once BuildKit's vendoring has the fix from https://github.com/moby/moby/pull/45942 - GO_VERSION: "1.20.5" + GO_VERSION: "1.20.6" DESTDIR: ./build jobs: diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index c0715d2564c9f..d91573c93f140 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -50,7 +50,7 @@ import ( ) func init() { - version.Version = "v0.11.6+0a15675913b7" + version.Version = "v0.11.6+616c3f613b54" } const labelCreatedAt = "buildkit/createdat" diff --git a/vendor.mod b/vendor.mod index d395f067930b3..0dac12a6799a9 100644 --- a/vendor.mod +++ b/vendor.mod @@ -56,7 +56,7 @@ require ( github.com/klauspost/compress v1.16.3 github.com/miekg/dns v1.1.43 github.com/mistifyio/go-zfs/v3 v3.0.1 - github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go + github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go github.com/moby/ipvs v1.1.0 github.com/moby/locker v1.0.1 github.com/moby/patternmatcher v0.5.0 diff --git a/vendor.sum b/vendor.sum index 4c9e9c9e08690..8dfeb13326506 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1043,8 +1043,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ= -github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 h1:9gjbrmALOUAJCqWL4RTwydPUiepMnkc3BNbBFiFiBeU= -github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7/go.mod h1:GCqKfHhz+pddzfgaR7WmHVEE3nKKZMMDPpK8mh3ZLv4= +github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 h1:LSh03Csyx/zQq8MreC9MYMQE/+5EkohwZMvXSS6kMZo= +github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54/go.mod h1:bMQDryngJKGvJ/ZuRFhrejurbvYSv3NkGCheQ59X4AM= github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= diff --git a/vendor/modules.txt b/vendor/modules.txt index 769853c517e9a..14e492185792b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -576,7 +576,7 @@ github.com/mistifyio/go-zfs/v3 # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 -# github.com/moby/buildkit v0.11.7-0.20230712171151-0a15675913b7 +# github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 ## explicit; go 1.18 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types From 35a8b00b187e43bda066045c4ca95d4ff594fe4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 24 Jul 2023 15:20:31 +0200 Subject: [PATCH 002/105] hack/integration: Add TEST_INTEGRATION_FAIL_FAST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this change, integration test would fail fast and not execute all test suites when one suite fails. Change this behavior into opt-in enabled by TEST_INTEGRATION_FAIL_FAST variable. Signed-off-by: Paweł Gronowski (cherry picked from commit 48cc28e4efc21be1d288c33a9e58417df2d28306) Signed-off-by: Paweł Gronowski --- Makefile | 1 + hack/make/.integration-test-helpers | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index aa1745369969e..ff7f713466de7 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,7 @@ DOCKER_ENVS := \ -e TEST_FORCE_VALIDATE \ -e TEST_INTEGRATION_DIR \ -e TEST_INTEGRATION_USE_SNAPSHOTTER \ + -e TEST_INTEGRATION_FAIL_FAST \ -e TEST_SKIP_INTEGRATION \ -e TEST_SKIP_INTEGRATION_CLI \ -e TESTCOVERAGE \ diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index b1c31b0b9062c..177a2ec8808be 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -66,6 +66,7 @@ run_test_integration() { run_test_integration_suites() { local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS" local dirs="$1" + local failed=0 for dir in ${dirs}; do if ! ( cd "$dir" @@ -96,8 +97,16 @@ run_test_integration_suites() { --junitfile="${ABS_DEST}/${pkgname//./-}-junit-report.xml" \ --raw-command \ -- go tool test2json -p "${pkgname}" -t ./test.main ${pkgtestflags} - ); then exit 1; fi + ); then + if [ -n "${TEST_INTEGRATION_FAIL_FAST}" ]; then + exit 1 + fi + failed=1 + fi done + if [ $failed -eq 1 ]; then + exit 1 + fi } build_test_suite_binaries() { From 3029f554cc26bff91504d6c4528179e0bc01eb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 13:28:18 +0200 Subject: [PATCH 003/105] c8d/readConfig: Translate c8d NotFound to errdefs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Gronowski (cherry picked from commit 7379d18018255069b03273589ad481b9a54d010b) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_list.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/daemon/containerd/image_list.go b/daemon/containerd/image_list.go index e8afa43945a46..319467a449fa6 100644 --- a/daemon/containerd/image_list.go +++ b/daemon/containerd/image_list.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" timetypes "github.com/docker/docker/api/types/time" + "github.com/docker/docker/errdefs" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -479,11 +480,20 @@ func computeSharedSize(chainIDs []digest.Digest, layers map[digest.Digest]int, s func readConfig(ctx context.Context, store content.Provider, desc ocispec.Descriptor, out interface{}) error { data, err := content.ReadBlob(ctx, store, desc) if err != nil { - return errors.Wrapf(err, "failed to read config content") + err = errors.Wrapf(err, "failed to read config content") + if cerrdefs.IsNotFound(err) { + return errdefs.NotFound(err) + } + return err } + err = json.Unmarshal(data, out) if err != nil { - return errors.Wrapf(err, "could not deserialize image config") + err = errors.Wrapf(err, "could not deserialize image config") + if cerrdefs.IsNotFound(err) { + return errdefs.NotFound(err) + } + return err } return nil From fcb68e55fa4ff6503d222483af41097f0c513595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 13:22:18 +0200 Subject: [PATCH 004/105] daemon/list: Replace ErrImageDoesNotExist check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check for generic `errdefs.NotFound` rather than specific error helper struct when checking if the error is caused by the image not being present. It still works for `ErrImageDoesNotExist` because it implements the NotFound errdefs interface too. Signed-off-by: Paweł Gronowski (cherry picked from commit 5a39bee63562490ba6f95c13fcac5fe1d46e330a) Signed-off-by: Paweł Gronowski --- daemon/list.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/daemon/list.go b/daemon/list.go index 676de6f22231e..003503ea96ee3 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -11,7 +11,6 @@ import ( "github.com/docker/docker/api/types/filters" imagetypes "github.com/docker/docker/api/types/image" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/images" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/go-connections/nat" @@ -585,7 +584,7 @@ func (daemon *Daemon) refreshImage(ctx context.Context, s *container.Snapshot, f tmpImage := s.Image // keep the original ref if still valid (hasn't changed) if tmpImage != s.ImageID { img, err := daemon.imageService.GetImage(ctx, tmpImage, imagetypes.GetImageOpts{}) - if _, isDNE := err.(images.ErrImageDoesNotExist); err != nil && !isDNE { + if err != nil && !errdefs.IsNotFound(err) { return nil, err } if err != nil || img.ImageID() != s.ImageID { From 6c4121a943eed9c42d051ae94d21419c9021a0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 13:20:20 +0200 Subject: [PATCH 005/105] daemon/list: Refactor refreshImage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add context comments and make it a bit more readable. Signed-off-by: Paweł Gronowski (cherry picked from commit 68991ae240ce56e302cc9297fcc4fa04d37f7c34) Signed-off-by: Paweł Gronowski --- daemon/list.go | 71 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/daemon/list.go b/daemon/list.go index 003503ea96ee3..5b90fbbede837 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -578,21 +578,70 @@ func includeContainerInList(container *container.Snapshot, filter *listContext) return includeContainer } -// refreshImage checks if the Image ref still points to the correct ID, and updates the ref to the actual ID when it doesn't +// refreshImage checks if the Image ref still points to the correct ID, and +// updates the ref to the actual ID when it doesn't. +// This happens when the image with a reference that was used to create +// container was deleted or updated and now resolves to a different ID. +// +// For example: +// $ docker run -d busybox:latest +// $ docker ps -a +// CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +// b0318bca5aef busybox "sh" 4 seconds ago Exited (0) 3 seconds ago ecstatic_beaver +// +// After some time, busybox image got updated on the Docker Hub: +// $ docker pull busybox:latest +// +// So now busybox:latest points to a different digest, but that doesn't impact +// the ecstatic_beaver container which was still created under an older +// version. In this case, it should still point to the original image ID it was +// created from. +// +// $ docker ps -a +// CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +// b0318bca5aef 3fbc63216742 "sh" 3 years ago Exited (0) 3 years ago ecstatic_beaver func (daemon *Daemon) refreshImage(ctx context.Context, s *container.Snapshot, filter *listContext) (*types.Container, error) { c := s.Container - tmpImage := s.Image // keep the original ref if still valid (hasn't changed) - if tmpImage != s.ImageID { - img, err := daemon.imageService.GetImage(ctx, tmpImage, imagetypes.GetImageOpts{}) - if err != nil && !errdefs.IsNotFound(err) { - return nil, err - } - if err != nil || img.ImageID() != s.ImageID { - // ref changed, we need to use original ID - tmpImage = s.ImageID + + // s.Image is the image reference passed by the user to create an image + // can be a: + // - name (like nginx, ubuntu:latest, docker.io/library/busybox:latest), + // - truncated ID (abcdef), + // - full digest (sha256:abcdef...) + // + // s.ImageID is the ID of the image that s.Image resolved to at the time + // of the container creation. It's always a full digest. + + // If these match, there's nothing to refresh. + if s.Image == s.ImageID { + return &c, nil + } + + // Check if the image reference still resolves to the same digest. + img, err := daemon.imageService.GetImage(ctx, s.Image, imagetypes.GetImageOpts{}) + + // If the image is no longer found or can't be resolved for some other + // reason. Update the Image to the specific ID of the original image it + // resolved to when the container was created. + if err != nil { + if !errdefs.IsNotFound(err) { + logrus.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "containerID": c.ID, + "image": s.Image, + "imageID": s.ImageID, + }).Warn("failed to resolve container image") } + c.Image = s.ImageID + return &c, nil } - c.Image = tmpImage + + // Also update the image to the specific image ID, if the Image now + // resolves to a different ID. + if img.ImageID() != s.ImageID { + c.Image = s.ImageID + } + return &c, nil } From 45ba926c6d1d26992971e4b215719e3ead369b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 14:52:11 +0200 Subject: [PATCH 006/105] daemon/list: Drop unused arg from containerReducer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refreshImage is the only function used as a reducer and it doesn't use the `filter *listContext`. Signed-off-by: Paweł Gronowski (cherry picked from commit 13180c1c4924ac2c8529507cf2405e3afe0a2413) Signed-off-by: Paweł Gronowski --- daemon/list.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/daemon/list.go b/daemon/list.go index 5b90fbbede837..5057e188d8840 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -41,7 +41,7 @@ type iterationAction int // containerReducer represents a reducer for a container. // Returns the object to serialize by the api. -type containerReducer func(context.Context, *container.Snapshot, *listContext) (*types.Container, error) +type containerReducer func(context.Context, *container.Snapshot) (*types.Container, error) const ( // includeContainer is the action to include a container in the reducer. @@ -230,7 +230,7 @@ func (daemon *Daemon) reducePsContainer(ctx context.Context, container *containe } // transform internal container struct into api structs - newC, err := reducer(ctx, container, filter) + newC, err := reducer(ctx, container) if err != nil { return nil, err } @@ -600,7 +600,7 @@ func includeContainerInList(container *container.Snapshot, filter *listContext) // $ docker ps -a // CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES // b0318bca5aef 3fbc63216742 "sh" 3 years ago Exited (0) 3 years ago ecstatic_beaver -func (daemon *Daemon) refreshImage(ctx context.Context, s *container.Snapshot, filter *listContext) (*types.Container, error) { +func (daemon *Daemon) refreshImage(ctx context.Context, s *container.Snapshot) (*types.Container, error) { c := s.Container // s.Image is the image reference passed by the user to create an image From 7927cae910adac0b821964a2c9d3784b10e5e2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 26 Jul 2023 12:09:27 +0200 Subject: [PATCH 007/105] c8d/container: Follow snapshot parents for size calculation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor GetContainerLayerSize to calculate unpacked image size only by following the snapshot parent tree directly instead of following it by using diff ids from image config. This works even if the original manifest/config used to create that container is no longer present in the content store. Signed-off-by: Paweł Gronowski (cherry picked from commit 4d8e3f54cc92dced080ff0b4daad23329cc25cc4) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_list.go | 52 ++++++------------- daemon/containerd/image_snapshot.go | 48 ++++++++++++++++++ daemon/containerd/service.go | 78 ++++++----------------------- 3 files changed, 79 insertions(+), 99 deletions(-) diff --git a/daemon/containerd/image_list.go b/daemon/containerd/image_list.go index e8afa43945a46..069a63ee56f38 100644 --- a/daemon/containerd/image_list.go +++ b/daemon/containerd/image_list.go @@ -10,6 +10,7 @@ import ( cerrdefs "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/labels" + "github.com/containerd/containerd/snapshots" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" @@ -142,41 +143,32 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions) func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, image *ImageManifest) (*types.ImageSummary, []digest.Digest, error) { diffIDs, err := image.RootFS(ctx) if err != nil { - return nil, nil, err - } - chainIDs := identity.ChainIDs(diffIDs) - - size, err := image.Size(ctx) - if err != nil { - return nil, nil, err + return nil, nil, errors.Wrapf(err, "failed to get rootfs of image %s", image.Name()) } // TODO(thaJeztah): do we need to take multiple snapshotters into account? See https://github.com/moby/moby/issues/45273 snapshotter := i.client.SnapshotService(i.snapshotter) - sizeCache := make(map[digest.Digest]int64) - snapshotSizeFn := func(d digest.Digest) (int64, error) { - if s, ok := sizeCache[d]; ok { - return s, nil - } - usage, err := snapshotter.Usage(ctx, d.String()) - if err != nil { - if cerrdefs.IsNotFound(err) { - return 0, nil - } - return 0, err + imageSnapshotID := identity.ChainID(diffIDs).String() + unpackedUsage, err := calculateSnapshotTotalUsage(ctx, snapshotter, imageSnapshotID) + if err != nil { + if !cerrdefs.IsNotFound(err) { + logrus.WithError(err).WithFields(logrus.Fields{ + "image": image.Name(), + "snapshotID": imageSnapshotID, + }).Warn("failed to calculate unpacked size of image") } - sizeCache[d] = usage.Size - return usage.Size, nil + unpackedUsage = snapshots.Usage{Size: 0} } - snapshotSize, err := computeSnapshotSize(chainIDs, snapshotSizeFn) + + contentSize, err := image.Size(ctx) if err != nil { return nil, nil, err } // totalSize is the size of the image's packed layers and snapshots // (unpacked layers) combined. - totalSize := size + snapshotSize + totalSize := contentSize + unpackedUsage.Size var repoTags, repoDigests []string rawImg := image.Metadata() @@ -225,7 +217,7 @@ func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore con Containers: -1, } - return summary, chainIDs, nil + return summary, identity.ChainIDs(diffIDs), nil } type imageFilterFunc func(image images.Image) bool @@ -446,20 +438,6 @@ func setupLabelFilter(store content.Store, fltrs filters.Args) (func(image image }, nil } -// computeSnapshotSize calculates the total size consumed by the snapshots -// for the given chainIDs. -func computeSnapshotSize(chainIDs []digest.Digest, sizeFn func(d digest.Digest) (int64, error)) (int64, error) { - var totalSize int64 - for _, chainID := range chainIDs { - size, err := sizeFn(chainID) - if err != nil { - return totalSize, err - } - totalSize += size - } - return totalSize, nil -} - func computeSharedSize(chainIDs []digest.Digest, layers map[digest.Digest]int, sizeFn func(d digest.Digest) (int64, error)) (int64, error) { var sharedSize int64 for _, chainID := range chainIDs { diff --git a/daemon/containerd/image_snapshot.go b/daemon/containerd/image_snapshot.go index a2152505c15c7..962c2100dc223 100644 --- a/daemon/containerd/image_snapshot.go +++ b/daemon/containerd/image_snapshot.go @@ -2,13 +2,18 @@ package containerd import ( "context" + "fmt" "github.com/containerd/containerd" + cerrdefs "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/snapshots" + "github.com/docker/docker/errdefs" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) // PrepareSnapshot prepares a snapshot from a parent image for a container @@ -67,3 +72,46 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentIma _, err = s.Prepare(ctx, id, parent) return err } + +// calculateSnapshotParentUsage returns the usage of all ancestors of the +// provided snapshot. It doesn't include the size of the snapshot itself. +func calculateSnapshotParentUsage(ctx context.Context, snapshotter snapshots.Snapshotter, snapshotID string) (snapshots.Usage, error) { + info, err := snapshotter.Stat(ctx, snapshotID) + if err != nil { + if cerrdefs.IsNotFound(err) { + return snapshots.Usage{}, errdefs.NotFound(err) + } + return snapshots.Usage{}, errdefs.System(errors.Wrapf(err, "snapshotter.Stat failed for %s", snapshotID)) + } + if info.Parent == "" { + return snapshots.Usage{}, errdefs.NotFound(fmt.Errorf("snapshot %s has no parent", snapshotID)) + } + + return calculateSnapshotTotalUsage(ctx, snapshotter, info.Parent) +} + +// calculateSnapshotTotalUsage returns the total usage of that snapshot +// including all of its ancestors. +func calculateSnapshotTotalUsage(ctx context.Context, snapshotter snapshots.Snapshotter, snapshotID string) (snapshots.Usage, error) { + var total snapshots.Usage + next := snapshotID + + for next != "" { + usage, err := snapshotter.Usage(ctx, next) + if err != nil { + if cerrdefs.IsNotFound(err) { + return total, errdefs.NotFound(errors.Wrapf(err, "non-existing ancestor of %s", snapshotID)) + } + return total, errdefs.System(errors.Wrapf(err, "snapshotter.Usage failed for %s", next)) + } + total.Size += usage.Size + total.Inodes += usage.Inodes + + info, err := snapshotter.Stat(ctx, next) + if err != nil { + return total, errdefs.System(errors.Wrapf(err, "snapshotter.Stat failed for %s", next)) + } + next = info.Parent + } + return total, nil +} diff --git a/daemon/containerd/service.go b/daemon/containerd/service.go index fce32665a845c..d68bcef91d77c 100644 --- a/daemon/containerd/service.go +++ b/daemon/containerd/service.go @@ -2,16 +2,15 @@ package containerd import ( "context" - "encoding/json" + "fmt" "sync/atomic" "github.com/containerd/containerd" - "github.com/containerd/containerd/content" + cerrdefs "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/snapshots" "github.com/docker/distribution/reference" - imagetypes "github.com/docker/docker/api/types/image" "github.com/docker/docker/container" daemonevents "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/images" @@ -20,8 +19,6 @@ import ( "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/registry" - "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -165,72 +162,29 @@ func (i *ImageService) GetContainerLayerSize(ctx context.Context, containerID st } snapshotter := i.client.SnapshotService(ctr.Driver) - - usage, err := snapshotter.Usage(ctx, containerID) + rwLayerUsage, err := snapshotter.Usage(ctx, containerID) if err != nil { - return 0, 0, err - } - - imageManifest, err := getContainerImageManifest(ctr) - if err != nil { - // Best efforts attempt to pick an image. - // We don't have platform information at this point, so we can only - // assume that the platform matches host. - // Otherwise this will give a wrong base image size (different - // platform), but should be close enough. - mfst, err := i.GetImageManifest(ctx, ctr.Config.Image, imagetypes.GetImageOpts{}) - if err != nil { - // Log error, don't error out whole operation. - logrus.WithFields(logrus.Fields{ - logrus.ErrorKey: err, - "container": containerID, - }).Warn("empty ImageManifest, can't calculate base image size") - return usage.Size, 0, nil + if cerrdefs.IsNotFound(err) { + return 0, 0, errdefs.NotFound(fmt.Errorf("rw layer snapshot not found for container %s", containerID)) } - imageManifest = *mfst - } - cs := i.client.ContentStore() - - imageManifestBytes, err := content.ReadBlob(ctx, cs, imageManifest) - if err != nil { - return 0, 0, err - } - - var manifest ocispec.Manifest - if err := json.Unmarshal(imageManifestBytes, &manifest); err != nil { - return 0, 0, err + return 0, 0, errdefs.System(errors.Wrapf(err, "snapshotter.Usage failed for %s", containerID)) } - imageConfigBytes, err := content.ReadBlob(ctx, cs, manifest.Config) + unpackedUsage, err := calculateSnapshotParentUsage(ctx, snapshotter, containerID) if err != nil { - return 0, 0, err - } - var img ocispec.Image - if err := json.Unmarshal(imageConfigBytes, &img); err != nil { - return 0, 0, err - } - - sizeCache := make(map[digest.Digest]int64) - snapshotSizeFn := func(d digest.Digest) (int64, error) { - if s, ok := sizeCache[d]; ok { - return s, nil + if cerrdefs.IsNotFound(err) { + logrus.WithField("ctr", containerID).Warn("parent of container snapshot no longer present") + } else { + logrus.WithError(err).WithField("ctr", containerID).Warn("unexpected error when calculating usage of the parent snapshots") } - u, err := snapshotter.Usage(ctx, d.String()) - if err != nil { - return 0, err - } - sizeCache[d] = u.Size - return u.Size, nil - } - - chainIDs := identity.ChainIDs(img.RootFS.DiffIDs) - snapShotSize, err := computeSnapshotSize(chainIDs, snapshotSizeFn) - if err != nil { - return 0, 0, err } + logrus.WithFields(logrus.Fields{ + "rwLayerUsage": rwLayerUsage.Size, + "unpacked": unpackedUsage.Size, + }).Debug("GetContainerLayerSize") // TODO(thaJeztah): include content-store size for the image (similar to "GET /images/json") - return usage.Size, usage.Size + snapShotSize, nil + return rwLayerUsage.Size, rwLayerUsage.Size + unpackedUsage.Size, nil } // getContainerImageManifest safely dereferences ImageManifest. From 3a6899c6fd7b99691495cc03031a72afacf4e077 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 28 Jul 2023 13:20:34 +0200 Subject: [PATCH 008/105] update containerd binary to v1.7.2 - full diff: https://github.com/containerd/containerd/compare/v1.7.1...v1.7.2 - release notes: https://github.com/containerd/containerd/releases/tag/v1.7.2 ---- Welcome to the v1.7.2 release of containerd! The second patch release for containerd 1.7 includes enhancements to CRI sandbox mode, Windows snapshot mounting support, and CRI and container IO bug fixes. CRI/Sandbox Updates - Publish sandbox events - Make stats respect sandbox's platform Other Notable Updates - Mount snapshots on Windows - Notify readiness when registered plugins are ready - Fix `cio.Cancel()` should close pipes - CDI: Use CRI `Config.CDIDevices` field for CDI injection Signed-off-by: Sebastiaan van Stijn (cherry picked from commit a78381c399447cd3406aabe37c933e7d7ecfacab) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- Dockerfile.windows | 2 +- hack/dockerfile/install/containerd.installer | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5413b699b9dc0..d0a6ed18697b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -197,7 +197,7 @@ RUN git init . && git remote add origin "https://github.com/containerd/container # When updating the binary version you may also need to update the vendor # version to pick up bug fixes or new APIs, however, usually the Go packages # are built from a commit from the master branch. -ARG CONTAINERD_VERSION=v1.7.1 +ARG CONTAINERD_VERSION=v1.7.2 RUN git fetch -q --depth 1 origin "${CONTAINERD_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD FROM base AS containerd-build diff --git a/Dockerfile.windows b/Dockerfile.windows index 68b4b74830a49..0bd492388c45e 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -168,7 +168,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref ARG GO_VERSION=1.20.6 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 -ARG CONTAINERD_VERSION=v1.7.1 +ARG CONTAINERD_VERSION=v1.7.2 # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. diff --git a/hack/dockerfile/install/containerd.installer b/hack/dockerfile/install/containerd.installer index bfdd27612fe45..e1ebc732f771c 100755 --- a/hack/dockerfile/install/containerd.installer +++ b/hack/dockerfile/install/containerd.installer @@ -15,7 +15,7 @@ set -e # the binary version you may also need to update the vendor version to pick up # bug fixes or new APIs, however, usually the Go packages are built from a # commit from the master branch. -: "${CONTAINERD_VERSION:=v1.7.1}" +: "${CONTAINERD_VERSION:=v1.7.2}" install_containerd() ( echo "Install containerd version $CONTAINERD_VERSION" From 02241b05fcc065b654f89e8f401a56ce35beb1e0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 28 Jul 2023 13:21:20 +0200 Subject: [PATCH 009/105] update containerd binary to v1.7.3 - full diff: https://github.com/containerd/containerd/compare/v1.7.2...v1.7.3 - release notes: https://github.com/containerd/containerd/releases/tag/v1.7.3 ---- Welcome to the v1.7.3 release of containerd! The third patch release for containerd 1.7 contains various fixes and updates. Notable Updates - RunC: Update runc binary to v1.1.8 - CRI: Fix `additionalGids`: it should fallback to `imageConfig.User` when `securityContext.RunAsUser`,`RunAsUsername` are empty - CRI: write generated CNI config atomically - Port-Forward: Correctly handle known errors - Resolve docker.NewResolver race condition - Fix `net.ipv4.ping_group_range` with userns - Runtime/V2/RunC: handle early exits w/o big locks - SecComp: always allow `name_to_handle_at` - CRI: Windows Pod Stats: Add a check to skip stats for containers that are not running - Task: don't `close()` io before cancel() - Remove CNI conf_template deprecation - Fix issue for HPC pod metrics Signed-off-by: Sebastiaan van Stijn (cherry picked from commit bf48d3ec29c1b48615c3a938d0144e8367e00dd3) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- Dockerfile.windows | 2 +- hack/dockerfile/install/containerd.installer | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d0a6ed18697b5..9a8e5ea7ffaaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -197,7 +197,7 @@ RUN git init . && git remote add origin "https://github.com/containerd/container # When updating the binary version you may also need to update the vendor # version to pick up bug fixes or new APIs, however, usually the Go packages # are built from a commit from the master branch. -ARG CONTAINERD_VERSION=v1.7.2 +ARG CONTAINERD_VERSION=v1.7.3 RUN git fetch -q --depth 1 origin "${CONTAINERD_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD FROM base AS containerd-build diff --git a/Dockerfile.windows b/Dockerfile.windows index 0bd492388c45e..e98f714e16c75 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -168,7 +168,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref ARG GO_VERSION=1.20.6 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 -ARG CONTAINERD_VERSION=v1.7.2 +ARG CONTAINERD_VERSION=v1.7.3 # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. diff --git a/hack/dockerfile/install/containerd.installer b/hack/dockerfile/install/containerd.installer index e1ebc732f771c..0e2b5f88c1c9d 100755 --- a/hack/dockerfile/install/containerd.installer +++ b/hack/dockerfile/install/containerd.installer @@ -15,7 +15,7 @@ set -e # the binary version you may also need to update the vendor version to pick up # bug fixes or new APIs, however, usually the Go packages are built from a # commit from the master branch. -: "${CONTAINERD_VERSION:=v1.7.2}" +: "${CONTAINERD_VERSION:=v1.7.3}" install_containerd() ( echo "Install containerd version $CONTAINERD_VERSION" From b6568d2dd5cdbd46d70f8a20378ad6c33da9852d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 28 Jul 2023 16:27:36 +0200 Subject: [PATCH 010/105] api/types/filters: fix errors not being matched by errors.Is() I found that the errors returned weren't matched with `errors.Is()` when wrapped. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 490fee7d45dfe69d313156b17d2e4ad12c2277ba) Signed-off-by: Sebastiaan van Stijn --- api/types/filters/parse.go | 10 +++---- api/types/filters/parse_test.go | 50 ++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/api/types/filters/parse.go b/api/types/filters/parse.go index 887648cf3e3d2..0c39ab5f18b53 100644 --- a/api/types/filters/parse.go +++ b/api/types/filters/parse.go @@ -98,7 +98,7 @@ func FromJSON(p string) (Args, error) { // Fallback to parsing arguments in the legacy slice format deprecated := map[string][]string{} if legacyErr := json.Unmarshal(raw, &deprecated); legacyErr != nil { - return args, invalidFilter{} + return args, &invalidFilter{} } args.fields = deprecatedArgs(deprecated) @@ -206,7 +206,7 @@ func (args Args) GetBoolOrDefault(key string, defaultValue bool) (bool, error) { } if len(fieldValues) == 0 { - return defaultValue, invalidFilter{key, nil} + return defaultValue, &invalidFilter{key, nil} } isFalse := fieldValues["0"] || fieldValues["false"] @@ -216,7 +216,7 @@ func (args Args) GetBoolOrDefault(key string, defaultValue bool) (bool, error) { invalid := !isFalse && !isTrue if conflicting || invalid { - return defaultValue, invalidFilter{key, args.Get(key)} + return defaultValue, &invalidFilter{key, args.Get(key)} } else if isFalse { return false, nil } else if isTrue { @@ -224,7 +224,7 @@ func (args Args) GetBoolOrDefault(key string, defaultValue bool) (bool, error) { } // This code shouldn't be reached. - return defaultValue, unreachableCode{Filter: key, Value: args.Get(key)} + return defaultValue, &unreachableCode{Filter: key, Value: args.Get(key)} } // ExactMatch returns true if the source matches exactly one of the values. @@ -282,7 +282,7 @@ func (args Args) Contains(field string) bool { func (args Args) Validate(accepted map[string]bool) error { for name := range args.fields { if !accepted[name] { - return invalidFilter{name, nil} + return &invalidFilter{name, nil} } } return nil diff --git a/api/types/filters/parse_test.go b/api/types/filters/parse_test.go index 74672ea2c99a7..5b1e2de307595 100644 --- a/api/types/filters/parse_test.go +++ b/api/types/filters/parse_test.go @@ -3,6 +3,7 @@ package filters // import "github.com/docker/docker/api/types/filters" import ( "encoding/json" "errors" + "fmt" "sort" "testing" @@ -95,15 +96,19 @@ func TestFromJSON(t *testing.T) { if err == nil { t.Fatalf("Expected an error with %v, got nothing", invalid) } - var invalidFilterError invalidFilter + var invalidFilterError *invalidFilter if !errors.As(err, &invalidFilterError) { t.Fatalf("Expected an invalidFilter error, got %T", err) } + wrappedErr := fmt.Errorf("something went wrong: %w", err) + if !errors.Is(wrappedErr, err) { + t.Errorf("Expected a wrapped error to be detected as invalidFilter") + } } for expectedArgs, matchers := range valid { - for _, json := range matchers { - args, err := FromJSON(json) + for _, jsonString := range matchers { + args, err := FromJSON(jsonString) if err != nil { t.Fatal(err) } @@ -358,9 +363,13 @@ func TestValidate(t *testing.T) { if err == nil { t.Fatal("Expected to return an error, got nil") } - var invalidFilterError invalidFilter + var invalidFilterError *invalidFilter if !errors.As(err, &invalidFilterError) { - t.Fatalf("Expected an invalidFilter error, got %T", err) + t.Errorf("Expected an invalidFilter error, got %T", err) + } + wrappedErr := fmt.Errorf("something went wrong: %w", err) + if !errors.Is(wrappedErr, err) { + t.Errorf("Expected a wrapped error to be detected as invalidFilter") } } @@ -421,7 +430,7 @@ func TestClone(t *testing.T) { } func TestGetBoolOrDefault(t *testing.T) { - for _, tC := range []struct { + for _, tc := range []struct { name string args map[string][]string defValue bool @@ -452,7 +461,7 @@ func TestGetBoolOrDefault(t *testing.T) { "dangling": {"potato"}, }, defValue: true, - expectedErr: invalidFilter{Filter: "dangling", Value: []string{"potato"}}, + expectedErr: &invalidFilter{Filter: "dangling", Value: []string{"potato"}}, expectedValue: true, }, { @@ -461,7 +470,7 @@ func TestGetBoolOrDefault(t *testing.T) { "dangling": {"banana", "potato"}, }, defValue: true, - expectedErr: invalidFilter{Filter: "dangling", Value: []string{"banana", "potato"}}, + expectedErr: &invalidFilter{Filter: "dangling", Value: []string{"banana", "potato"}}, expectedValue: true, }, { @@ -470,7 +479,7 @@ func TestGetBoolOrDefault(t *testing.T) { "dangling": {"false", "true"}, }, defValue: false, - expectedErr: invalidFilter{Filter: "dangling", Value: []string{"false", "true"}}, + expectedErr: &invalidFilter{Filter: "dangling", Value: []string{"false", "true"}}, expectedValue: false, }, { @@ -479,7 +488,7 @@ func TestGetBoolOrDefault(t *testing.T) { "dangling": {"false", "true", "1"}, }, defValue: true, - expectedErr: invalidFilter{Filter: "dangling", Value: []string{"false", "true", "1"}}, + expectedErr: &invalidFilter{Filter: "dangling", Value: []string{"false", "true", "1"}}, expectedValue: true, }, { @@ -501,35 +510,38 @@ func TestGetBoolOrDefault(t *testing.T) { expectedValue: false, }, } { - tC := tC - t.Run(tC.name, func(t *testing.T) { + tc := tc + t.Run(tc.name, func(t *testing.T) { a := NewArgs() - for key, values := range tC.args { + for key, values := range tc.args { for _, value := range values { a.Add(key, value) } } - value, err := a.GetBoolOrDefault("dangling", tC.defValue) + value, err := a.GetBoolOrDefault("dangling", tc.defValue) - if tC.expectedErr == nil { + if tc.expectedErr == nil { assert.Check(t, is.Nil(err)) } else { - assert.Check(t, is.ErrorType(err, tC.expectedErr)) + assert.Check(t, is.ErrorType(err, tc.expectedErr)) // Check if error is the same. - expected := tC.expectedErr.(invalidFilter) - actual := err.(invalidFilter) + expected := tc.expectedErr.(*invalidFilter) + actual := err.(*invalidFilter) assert.Check(t, is.Equal(expected.Filter, actual.Filter)) sort.Strings(expected.Value) sort.Strings(actual.Value) assert.Check(t, is.DeepEqual(expected.Value, actual.Value)) + + wrappedErr := fmt.Errorf("something went wrong: %w", err) + assert.Check(t, errors.Is(wrappedErr, err), "Expected a wrapped error to be detected as invalidFilter") } - assert.Check(t, is.Equal(tC.expectedValue, value)) + assert.Check(t, is.Equal(tc.expectedValue, value)) }) } From 6be708aa7d02727598fed6fe73424a90d5ecd82d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 29 Jul 2023 19:07:09 +0200 Subject: [PATCH 011/105] vendor: gotest.tools/v3 v3.5.0 - go.mod: update dependencies and go version by - Use Go1.20 - Fix couple of typos - Added `WithStdout` and `WithStderr` helpers - Moved `cmdOperators` handling from `RunCmd` to `StartCmd` - Deprecate `assert.ErrorType` - Remove outdated Dockerfile - add godoc links full diff: https://github.com/gotestyourself/gotest.tools/compare/v3.4.0...v3.5.0 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ce053a14aa3fd28ae7225d67220b177ade75bc0c) Signed-off-by: Sebastiaan van Stijn --- hack/validate/golangci-lint.yml | 4 ++ vendor.mod | 2 +- vendor.sum | 4 +- vendor/gotest.tools/v3/assert/assert.go | 69 ++++++++++---------- vendor/gotest.tools/v3/assert/cmp/compare.go | 25 +++---- vendor/gotest.tools/v3/assert/cmp/result.go | 12 ++-- vendor/gotest.tools/v3/assert/opt/opt.go | 12 ++-- vendor/gotest.tools/v3/env/env.go | 2 +- vendor/gotest.tools/v3/icmd/command.go | 17 +++-- vendor/gotest.tools/v3/icmd/ops.go | 14 ++++ vendor/gotest.tools/v3/poll/check.go | 4 +- vendor/gotest.tools/v3/poll/poll.go | 18 ++--- vendor/modules.txt | 4 +- 13 files changed, 106 insertions(+), 81 deletions(-) diff --git a/hack/validate/golangci-lint.yml b/hack/validate/golangci-lint.yml index ab188068506df..2832ad7e77352 100644 --- a/hack/validate/golangci-lint.yml +++ b/hack/validate/golangci-lint.yml @@ -126,6 +126,10 @@ issues: - text: "SA1019: httputil.ErrPersistEOF" linters: - staticcheck + # FIXME temporarily suppress these (see https://github.com/gotestyourself/gotest.tools/issues/272) + - text: "SA1019: (assert|cmp|is)\\.ErrorType is deprecated" + linters: + - staticcheck # Maximum issues count per one linter. Set to 0 to disable. Default is 50. max-issues-per-linter: 0 diff --git a/vendor.mod b/vendor.mod index 0dac12a6799a9..93b49543cb836 100644 --- a/vendor.mod +++ b/vendor.mod @@ -94,7 +94,7 @@ require ( golang.org/x/time v0.3.0 google.golang.org/genproto v0.0.0-20220706185917-7780775163c4 google.golang.org/grpc v1.50.1 - gotest.tools/v3 v3.4.0 + gotest.tools/v3 v3.5.0 resenje.org/singleflight v0.3.0 ) diff --git a/vendor.sum b/vendor.sum index 8dfeb13326506..7a265df2c3284 100644 --- a/vendor.sum +++ b/vendor.sum @@ -2223,8 +2223,8 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= -gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/gotest.tools/v3/assert/assert.go b/vendor/gotest.tools/v3/assert/assert.go index f75f9f510e6b9..c418bd07b5c43 100644 --- a/vendor/gotest.tools/v3/assert/assert.go +++ b/vendor/gotest.tools/v3/assert/assert.go @@ -4,7 +4,7 @@ values in tests. When an assertion fails a helpful error message is printed. # Example usage -All the assertions in this package use testing.T.Helper to mark themselves as +All the assertions in this package use [testing.T.Helper] to mark themselves as test helpers. This allows the testing package to print the filename and line number of the file function that failed. @@ -67,19 +67,19 @@ message is omitted from these examples for brevity. # Assert and Check -Assert and Check are very similar, they both accept a Comparison, and fail +[Assert] and [Check] are very similar, they both accept a [cmp.Comparison], and fail the test when the comparison fails. The one difference is that Assert uses -testing.T.FailNow to fail the test, which will end the test execution immediately. -Check uses testing.T.Fail to fail the test, which allows it to return the +[testing.T.FailNow] to fail the test, which will end the test execution immediately. +Check uses [testing.T.Fail] to fail the test, which allows it to return the result of the comparison, then proceed with the rest of the test case. -Like testing.T.FailNow, Assert must be called from the goroutine running the test, -not from other goroutines created during the test. Check is safe to use from any +Like [testing.T.FailNow], [Assert] must be called from the goroutine running the test, +not from other goroutines created during the test. [Check] is safe to use from any goroutine. # Comparisons -Package http://pkg.go.dev/gotest.tools/v3/assert/cmp provides +Package [gotest.tools/v3/assert/cmp] provides many common comparisons. Additional comparisons can be written to compare values in other ways. See the example Assert (CustomComparison). @@ -98,11 +98,11 @@ import ( "gotest.tools/v3/internal/assert" ) -// BoolOrComparison can be a bool, cmp.Comparison, or error. See Assert for +// BoolOrComparison can be a bool, [cmp.Comparison], or error. See [Assert] for // details about how this type is used. type BoolOrComparison interface{} -// TestingT is the subset of testing.T used by the assert package. +// TestingT is the subset of [testing.T] (see also [testing.TB]) used by the assert package. type TestingT interface { FailNow() Fail() @@ -133,11 +133,11 @@ type helperT interface { // // Extra details can be added to the failure message using msgAndArgs. msgAndArgs // may be either a single string, or a format string and args that will be -// passed to fmt.Sprintf. +// passed to [fmt.Sprintf]. // -// Assert uses t.FailNow to fail the test. Like t.FailNow, Assert must be called +// Assert uses [testing.TB.FailNow] to fail the test. Like t.FailNow, Assert must be called // from the goroutine running the test function, not from other -// goroutines created during the test. Use Check from other goroutines. +// goroutines created during the test. Use [Check] from other goroutines. func Assert(t TestingT, comparison BoolOrComparison, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { ht.Helper() @@ -151,7 +151,7 @@ func Assert(t TestingT, comparison BoolOrComparison, msgAndArgs ...interface{}) // failed, a failure message is printed, and Check returns false. If the comparison // is successful Check returns true. Check may be called from any goroutine. // -// See Assert for details about the comparison arg and failure messages. +// See [Assert] for details about the comparison arg and failure messages. func Check(t TestingT, comparison BoolOrComparison, msgAndArgs ...interface{}) bool { if ht, ok := t.(helperT); ok { ht.Helper() @@ -166,9 +166,9 @@ func Check(t TestingT, comparison BoolOrComparison, msgAndArgs ...interface{}) b // NilError fails the test immediately if err is not nil, and includes err.Error // in the failure message. // -// NilError uses t.FailNow to fail the test. Like t.FailNow, NilError must be +// NilError uses [testing.TB.FailNow] to fail the test. Like t.FailNow, NilError must be // called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check from other goroutines. +// goroutines created during the test. Use [Check] from other goroutines. func NilError(t TestingT, err error, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { ht.Helper() @@ -193,9 +193,9 @@ func NilError(t TestingT, err error, msgAndArgs ...interface{}) { // the unified diff will be augmented by replacing whitespace characters with // visible characters to identify the whitespace difference. // -// Equal uses t.FailNow to fail the test. Like t.FailNow, Equal must be +// Equal uses [testing.T.FailNow] to fail the test. Like t.FailNow, Equal must be // called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.Equal from other +// goroutines created during the test. Use [Check] with [cmp.Equal] from other // goroutines. func Equal(t TestingT, x, y interface{}, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { @@ -206,15 +206,15 @@ func Equal(t TestingT, x, y interface{}, msgAndArgs ...interface{}) { } } -// DeepEqual uses google/go-cmp (https://godoc.org/github.com/google/go-cmp/cmp) +// DeepEqual uses [github.com/google/go-cmp/cmp] // to assert two values are equal and fails the test if they are not equal. // -// Package http://pkg.go.dev/gotest.tools/v3/assert/opt provides some additional +// Package [gotest.tools/v3/assert/opt] provides some additional // commonly used Options. // -// DeepEqual uses t.FailNow to fail the test. Like t.FailNow, DeepEqual must be +// DeepEqual uses [testing.T.FailNow] to fail the test. Like t.FailNow, DeepEqual must be // called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.DeepEqual from other +// goroutines created during the test. Use [Check] with [cmp.DeepEqual] from other // goroutines. func DeepEqual(t TestingT, x, y interface{}, opts ...gocmp.Option) { if ht, ok := t.(helperT); ok { @@ -227,13 +227,13 @@ func DeepEqual(t TestingT, x, y interface{}, opts ...gocmp.Option) { // Error fails the test if err is nil, or if err.Error is not equal to expected. // Both err.Error and expected will be included in the failure message. -// Error performs an exact match of the error text. Use ErrorContains if only -// part of the error message is relevant. Use ErrorType or ErrorIs to compare +// Error performs an exact match of the error text. Use [ErrorContains] if only +// part of the error message is relevant. Use [ErrorType] or [ErrorIs] to compare // errors by type. // -// Error uses t.FailNow to fail the test. Like t.FailNow, Error must be +// Error uses [testing.T.FailNow] to fail the test. Like t.FailNow, Error must be // called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.Error from other +// goroutines created during the test. Use [Check] with [cmp.Error] from other // goroutines. func Error(t TestingT, err error, expected string, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { @@ -248,9 +248,9 @@ func Error(t TestingT, err error, expected string, msgAndArgs ...interface{}) { // contain the expected substring. Both err.Error and the expected substring // will be included in the failure message. // -// ErrorContains uses t.FailNow to fail the test. Like t.FailNow, ErrorContains +// ErrorContains uses [testing.T.FailNow] to fail the test. Like t.FailNow, ErrorContains // must be called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.ErrorContains from other +// goroutines created during the test. Use [Check] with [cmp.ErrorContains] from other // goroutines. func ErrorContains(t TestingT, err error, substring string, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { @@ -262,8 +262,7 @@ func ErrorContains(t TestingT, err error, substring string, msgAndArgs ...interf } // ErrorType fails the test if err is nil, or err is not the expected type. -// Most new code should use ErrorIs instead. ErrorType may be deprecated in the -// future. +// New code should use ErrorIs instead. // // Expected can be one of: // @@ -281,10 +280,12 @@ func ErrorContains(t TestingT, err error, substring string, msgAndArgs ...interf // reflect.Type // The assertion fails if err does not implement the reflect.Type. // -// ErrorType uses t.FailNow to fail the test. Like t.FailNow, ErrorType +// ErrorType uses [testing.T.FailNow] to fail the test. Like t.FailNow, ErrorType // must be called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.ErrorType from other +// goroutines created during the test. Use [Check] with [cmp.ErrorType] from other // goroutines. +// +// Deprecated: Use [ErrorIs] func ErrorType(t TestingT, err error, expected interface{}, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { ht.Helper() @@ -295,12 +296,12 @@ func ErrorType(t TestingT, err error, expected interface{}, msgAndArgs ...interf } // ErrorIs fails the test if err is nil, or the error does not match expected -// when compared using errors.Is. See https://golang.org/pkg/errors/#Is for +// when compared using errors.Is. See [errors.Is] for // accepted arguments. // -// ErrorIs uses t.FailNow to fail the test. Like t.FailNow, ErrorIs +// ErrorIs uses [testing.T.FailNow] to fail the test. Like t.FailNow, ErrorIs // must be called from the goroutine running the test function, not from other -// goroutines created during the test. Use Check with cmp.ErrorIs from other +// goroutines created during the test. Use [Check] with [cmp.ErrorIs] from other // goroutines. func ErrorIs(t TestingT, err error, expected error, msgAndArgs ...interface{}) { if ht, ok := t.(helperT); ok { diff --git a/vendor/gotest.tools/v3/assert/cmp/compare.go b/vendor/gotest.tools/v3/assert/cmp/compare.go index 4112b00433f22..118844f35d071 100644 --- a/vendor/gotest.tools/v3/assert/cmp/compare.go +++ b/vendor/gotest.tools/v3/assert/cmp/compare.go @@ -12,17 +12,16 @@ import ( "gotest.tools/v3/internal/format" ) -// Comparison is a function which compares values and returns ResultSuccess if +// Comparison is a function which compares values and returns [ResultSuccess] if // the actual value matches the expected value. If the values do not match the -// Result will contain a message about why it failed. +// [Result] will contain a message about why it failed. type Comparison func() Result -// DeepEqual compares two values using google/go-cmp -// (https://godoc.org/github.com/google/go-cmp/cmp) +// DeepEqual compares two values using [github.com/google/go-cmp/cmp] // and succeeds if the values are equal. // // The comparison can be customized using comparison Options. -// Package http://pkg.go.dev/gotest.tools/v3/assert/opt provides some additional +// Package [gotest.tools/v3/assert/opt] provides some additional // commonly used Options. func DeepEqual(x, y interface{}, opts ...cmp.Option) Comparison { return func() (result Result) { @@ -61,7 +60,7 @@ func toResult(success bool, msg string) Result { return ResultFailure(msg) } -// RegexOrPattern may be either a *regexp.Regexp or a string that is a valid +// RegexOrPattern may be either a [*regexp.Regexp] or a string that is a valid // regexp pattern. type RegexOrPattern interface{} @@ -95,7 +94,7 @@ func Regexp(re RegexOrPattern, v string) Comparison { } } -// Equal succeeds if x == y. See assert.Equal for full documentation. +// Equal succeeds if x == y. See [gotest.tools/v3/assert.Equal] for full documentation. func Equal(x, y interface{}) Comparison { return func() Result { switch { @@ -159,10 +158,10 @@ func Len(seq interface{}, expected int) Comparison { // slice, or array. // // If collection is a string, item must also be a string, and is compared using -// strings.Contains(). +// [strings.Contains]. // If collection is a Map, contains will succeed if item is a key in the map. // If collection is a slice or array, item is compared to each item in the -// sequence using reflect.DeepEqual(). +// sequence using [reflect.DeepEqual]. func Contains(collection interface{}, item interface{}) Comparison { return func() Result { colValue := reflect.ValueOf(collection) @@ -259,7 +258,7 @@ func formatErrorMessage(err error) string { // Nil succeeds if obj is a nil interface, pointer, or function. // -// Use NilError() for comparing errors. Use Len(obj, 0) for comparing slices, +// Use [gotest.tools/v3/assert.NilError] for comparing errors. Use Len(obj, 0) for comparing slices, // maps, and channels. func Nil(obj interface{}) Comparison { msgFunc := func(value reflect.Value) string { @@ -306,7 +305,9 @@ func isNil(obj interface{}, msgFunc func(reflect.Value) string) Comparison { // // reflect.Type // -// Fails if err does not implement the reflect.Type +// Fails if err does not implement the [reflect.Type]. +// +// Deprecated: Use [ErrorIs] func ErrorType(err error, expected interface{}) Comparison { return func() Result { switch expectedType := expected.(type) { @@ -381,7 +382,7 @@ var ( ) // ErrorIs succeeds if errors.Is(actual, expected) returns true. See -// https://golang.org/pkg/errors/#Is for accepted argument values. +// [errors.Is] for accepted argument values. func ErrorIs(actual error, expected error) Comparison { return func() Result { if errors.Is(actual, expected) { diff --git a/vendor/gotest.tools/v3/assert/cmp/result.go b/vendor/gotest.tools/v3/assert/cmp/result.go index 28ef8d3d46c5d..9992ede54465b 100644 --- a/vendor/gotest.tools/v3/assert/cmp/result.go +++ b/vendor/gotest.tools/v3/assert/cmp/result.go @@ -10,12 +10,12 @@ import ( "gotest.tools/v3/internal/source" ) -// A Result of a Comparison. +// A Result of a [Comparison]. type Result interface { Success() bool } -// StringResult is an implementation of Result that reports the error message +// StringResult is an implementation of [Result] that reports the error message // string verbatim and does not provide any templating or formatting of the // message. type StringResult struct { @@ -34,16 +34,16 @@ func (r StringResult) FailureMessage() string { return r.message } -// ResultSuccess is a constant which is returned by a ComparisonWithResult to +// ResultSuccess is a constant which is returned by a [Comparison] to // indicate success. var ResultSuccess = StringResult{success: true} -// ResultFailure returns a failed Result with a failure message. +// ResultFailure returns a failed [Result] with a failure message. func ResultFailure(message string) StringResult { return StringResult{message: message} } -// ResultFromError returns ResultSuccess if err is nil. Otherwise ResultFailure +// ResultFromError returns [ResultSuccess] if err is nil. Otherwise [ResultFailure] // is returned with the error message as the failure message. func ResultFromError(err error) Result { if err == nil { @@ -74,7 +74,7 @@ func (r templatedResult) UpdatedExpected(stackIndex int) error { return source.UpdateExpectedValue(stackIndex+1, r.data["x"], r.data["y"]) } -// ResultFailureTemplate returns a Result with a template string and data which +// ResultFailureTemplate returns a [Result] with a template string and data which // can be used to format a failure message. The template may access data from .Data, // the comparison args with the callArg function, and the formatNode function may // be used to format the call args. diff --git a/vendor/gotest.tools/v3/assert/opt/opt.go b/vendor/gotest.tools/v3/assert/opt/opt.go index 357cdf2ebae1f..bd4c9dc3a2e8f 100644 --- a/vendor/gotest.tools/v3/assert/opt/opt.go +++ b/vendor/gotest.tools/v3/assert/opt/opt.go @@ -11,7 +11,7 @@ import ( gocmp "github.com/google/go-cmp/cmp" ) -// DurationWithThreshold returns a gocmp.Comparer for comparing time.Duration. The +// DurationWithThreshold returns a [gocmp.Comparer] for comparing [time.Duration]. The // Comparer returns true if the difference between the two Duration values is // within the threshold and neither value is zero. func DurationWithThreshold(threshold time.Duration) gocmp.Option { @@ -28,7 +28,7 @@ func cmpDuration(threshold time.Duration) func(x, y time.Duration) bool { } } -// TimeWithThreshold returns a gocmp.Comparer for comparing time.Time. The +// TimeWithThreshold returns a [gocmp.Comparer] for comparing [time.Time]. The // Comparer returns true if the difference between the two Time values is // within the threshold and neither value is zero. func TimeWithThreshold(threshold time.Duration) gocmp.Option { @@ -45,12 +45,12 @@ func cmpTime(threshold time.Duration) func(x, y time.Time) bool { } } -// PathString is a gocmp.FilterPath filter that returns true when path.String() +// PathString is a [gocmp.FilterPath] filter that returns true when path.String() // matches any of the specs. // // The path spec is a dot separated string where each segment is a field name. // Slices, Arrays, and Maps are always matched against every element in the -// sequence. gocmp.Indirect, gocmp.Transform, and gocmp.TypeAssertion are always +// sequence. [gocmp.Indirect], [gocmp.Transform], and [gocmp.TypeAssertion] are always // ignored. // // Note: this path filter is not type safe. Incorrect paths will be silently @@ -66,7 +66,7 @@ func PathString(specs ...string) func(path gocmp.Path) bool { } } -// PathDebug is a gocmp.FilerPath filter that always returns false. It prints +// PathDebug is a [gocmp.FilterPath] filter that always returns false. It prints // each path it receives. It can be used to debug path matching problems. func PathDebug(path gocmp.Path) bool { fmt.Printf("PATH string=%s gostring=%s\n", path, path.GoString()) @@ -95,7 +95,7 @@ func stepTypeFields(step gocmp.PathStep) string { return "" } -// PathField is a gocmp.FilerPath filter that matches a struct field by name. +// PathField is a [gocmp.FilterPath] filter that matches a struct field by name. // PathField will match every instance of the field in a recursive or nested // structure. func PathField(structType interface{}, field string) func(gocmp.Path) bool { diff --git a/vendor/gotest.tools/v3/env/env.go b/vendor/gotest.tools/v3/env/env.go index 71efc39307a57..9653cf1875edc 100644 --- a/vendor/gotest.tools/v3/env/env.go +++ b/vendor/gotest.tools/v3/env/env.go @@ -72,7 +72,7 @@ func PatchAll(t assert.TestingT, env map[string]string) func() { return clean } -// ToMap takes a list of strings in the format returned by os.Environ() and +// ToMap takes a list of strings in the format returned by [os.Environ] and // returns a mapping of keys to values. func ToMap(env []string) map[string]string { result := map[string]string{} diff --git a/vendor/gotest.tools/v3/icmd/command.go b/vendor/gotest.tools/v3/icmd/command.go index a15834bab408e..a3e167a013348 100644 --- a/vendor/gotest.tools/v3/icmd/command.go +++ b/vendor/gotest.tools/v3/icmd/command.go @@ -195,6 +195,7 @@ type Cmd struct { Timeout time.Duration Stdin io.Reader Stdout io.Writer + Stderr io.Writer Dir string Env []string ExtraFiles []*os.File @@ -207,10 +208,7 @@ func Command(command string, args ...string) Cmd { // RunCmd runs a command and returns a Result func RunCmd(cmd Cmd, cmdOperators ...CmdOp) *Result { - for _, op := range cmdOperators { - op(&cmd) - } - result := StartCmd(cmd) + result := StartCmd(cmd, cmdOperators...) if result.Error != nil { return result } @@ -223,7 +221,10 @@ func RunCommand(command string, args ...string) *Result { } // StartCmd starts a command, but doesn't wait for it to finish -func StartCmd(cmd Cmd) *Result { +func StartCmd(cmd Cmd, cmdOperators ...CmdOp) *Result { + for _, op := range cmdOperators { + op(&cmd) + } result := buildCmd(cmd) if result.Error != nil { return result @@ -252,7 +253,11 @@ func buildCmd(cmd Cmd) *Result { } else { execCmd.Stdout = outBuffer } - execCmd.Stderr = errBuffer + if cmd.Stderr != nil { + execCmd.Stderr = io.MultiWriter(errBuffer, cmd.Stderr) + } else { + execCmd.Stderr = errBuffer + } execCmd.ExtraFiles = cmd.ExtraFiles return &Result{ diff --git a/vendor/gotest.tools/v3/icmd/ops.go b/vendor/gotest.tools/v3/icmd/ops.go index 35c3958d52bb3..aa3bc1e8f8eec 100644 --- a/vendor/gotest.tools/v3/icmd/ops.go +++ b/vendor/gotest.tools/v3/icmd/ops.go @@ -38,6 +38,20 @@ func WithStdin(r io.Reader) CmdOp { } } +// WithStdout sets the standard output of the command to the specified writer +func WithStdout(w io.Writer) CmdOp { + return func(c *Cmd) { + c.Stdout = w + } +} + +// WithStderr sets the standard error of the command to the specified writer +func WithStderr(w io.Writer) CmdOp { + return func(c *Cmd) { + c.Stderr = w + } +} + // WithExtraFile adds a file descriptor to the command func WithExtraFile(f *os.File) CmdOp { return func(c *Cmd) { diff --git a/vendor/gotest.tools/v3/poll/check.go b/vendor/gotest.tools/v3/poll/check.go index 46880f5b25cae..fa0f21c1e17c8 100644 --- a/vendor/gotest.tools/v3/poll/check.go +++ b/vendor/gotest.tools/v3/poll/check.go @@ -5,7 +5,7 @@ import ( "os" ) -// Check is a function which will be used as check for the WaitOn method. +// Check is a function which will be used as check for the [WaitOn] method. type Check func(t LogT) Result // FileExists looks on filesystem and check that path exists. @@ -29,7 +29,7 @@ func FileExists(path string) Check { } // Connection try to open a connection to the address on the -// named network. See net.Dial for a description of the network and +// named network. See [net.Dial] for a description of the network and // address parameters. func Connection(network, address string) Check { return func(t LogT) Result { diff --git a/vendor/gotest.tools/v3/poll/poll.go b/vendor/gotest.tools/v3/poll/poll.go index 29c5b40e187fd..cfd6d43ace742 100644 --- a/vendor/gotest.tools/v3/poll/poll.go +++ b/vendor/gotest.tools/v3/poll/poll.go @@ -11,13 +11,13 @@ import ( "gotest.tools/v3/internal/assert" ) -// TestingT is the subset of testing.T used by WaitOn +// TestingT is the subset of [testing.T] used by [WaitOn] type TestingT interface { LogT Fatalf(format string, args ...interface{}) } -// LogT is a logging interface that is passed to the WaitOn check function +// LogT is a logging interface that is passed to the [WaitOn] check function type LogT interface { Log(args ...interface{}) Logf(format string, args ...interface{}) @@ -27,7 +27,7 @@ type helperT interface { Helper() } -// Settings are used to configure the behaviour of WaitOn +// Settings are used to configure the behaviour of [WaitOn] type Settings struct { // Timeout is the maximum time to wait for the condition. Defaults to 10s. Timeout time.Duration @@ -57,7 +57,7 @@ func WithTimeout(timeout time.Duration) SettingOp { } } -// Result of a check performed by WaitOn +// Result of a check performed by [WaitOn] type Result interface { // Error indicates that the check failed and polling should stop, and the // the has failed @@ -86,20 +86,20 @@ func (r result) Error() error { return r.err } -// Continue returns a Result that indicates to WaitOn that it should continue +// Continue returns a [Result] that indicates to [WaitOn] that it should continue // polling. The message text will be used as the failure message if the timeout // is reached. func Continue(message string, args ...interface{}) Result { return result{message: fmt.Sprintf(message, args...)} } -// Success returns a Result where Done() returns true, which indicates to WaitOn +// Success returns a [Result] where Done() returns true, which indicates to [WaitOn] // that it should stop polling and exit without an error. func Success() Result { return result{done: true} } -// Error returns a Result that indicates to WaitOn that it should fail the test +// Error returns a [Result] that indicates to [WaitOn] that it should fail the test // and stop polling. func Error(err error) Result { return result{err: err} @@ -143,9 +143,9 @@ func WaitOn(t TestingT, check Check, pollOps ...SettingOp) { } } -// Compare values using the cmp.Comparison. If the comparison fails return a +// Compare values using the [cmp.Comparison]. If the comparison fails return a // result which indicates to WaitOn that it should continue waiting. -// If the comparison is successful then WaitOn stops polling. +// If the comparison is successful then [WaitOn] stops polling. func Compare(compare cmp.Comparison) Result { buf := new(logBuffer) if assert.RunComparison(buf, assert.ArgsAtZeroIndex, compare) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 14e492185792b..65ad85f86de6e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1284,8 +1284,8 @@ google.golang.org/protobuf/types/known/fieldmaskpb google.golang.org/protobuf/types/known/structpb google.golang.org/protobuf/types/known/timestamppb google.golang.org/protobuf/types/known/wrapperspb -# gotest.tools/v3 v3.4.0 -## explicit; go 1.13 +# gotest.tools/v3 v3.5.0 +## explicit; go 1.17 gotest.tools/v3/assert gotest.tools/v3/assert/cmp gotest.tools/v3/assert/opt From a6f8e973422abb8dfc47be6d229798b7a75f9674 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 11 Mar 2021 18:57:01 +0000 Subject: [PATCH 012/105] Improve test daemon logging 1. On failed start tail the daemon logs 2. Exposes generic tailing functions to make test debugging simpler Signed-off-by: Brian Goff (cherry picked from commit 914888cf8bfb7a25f8e8018f753d923d3495a0b3) Signed-off-by: Sebastiaan van Stijn --- testutil/daemon/daemon.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/testutil/daemon/daemon.go b/testutil/daemon/daemon.go index 98230960c6da0..f0b5537b3089d 100644 --- a/testutil/daemon/daemon.go +++ b/testutil/daemon/daemon.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/container" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/pkg/tailfile" "github.com/docker/docker/testutil/request" "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" @@ -296,10 +297,41 @@ func (d *Daemon) Cleanup(t testing.TB) { cleanupNetworkNamespace(t, d) } +// TailLogsT attempts to tail N lines from the daemon logs. +// If there is an error the error is only logged, it does not cause an error with the test. +func (d *Daemon) TailLogsT(t LogT, n int) { + lines, err := d.TailLogs(n) + if err != nil { + t.Logf("[%s] %v", d.id, err) + return + } + for _, l := range lines { + t.Logf("[%s] %s", d.id, string(l)) + } +} + +// TailLogs tails N lines from the daemon logs +func (d *Daemon) TailLogs(n int) ([][]byte, error) { + logF, err := os.Open(d.logFile.Name()) + if err != nil { + return nil, errors.Wrap(err, "error opening daemon log file after failed start") + } + + defer logF.Close() + lines, err := tailfile.TailFile(logF, n) + if err != nil { + return nil, errors.Wrap(err, "error tailing log daemon logs") + } + + return lines, nil + +} + // Start starts the daemon and return once it is ready to receive requests. func (d *Daemon) Start(t testing.TB, args ...string) { t.Helper() if err := d.StartWithError(args...); err != nil { + d.TailLogsT(t, 20) d.DumpStackAndQuit() // in case the daemon is stuck t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err) } From d6536d44e972dd9fe9c632691a1bb7232264653b Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 28 Jul 2023 19:50:24 +0000 Subject: [PATCH 013/105] TestDaemonProxy: check proxy settings early Allows tests to report their proxy settings for easier troubleshooting on failures. Signed-off-by: Brian Goff (cherry picked from commit 8197752d681ec700ceee6f5d71f9cb1fec2adf19) Signed-off-by: Sebastiaan van Stijn --- integration/daemon/daemon_test.go | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index b6e00c81b9c04..28c5c422cd41e 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -190,6 +190,12 @@ func TestDaemonProxy(t *testing.T) { defer func() { _ = c.Close() }() ctx := context.Background() d.Start(t) + defer d.Stop(t) + + info := d.Info(t) + assert.Check(t, is.Equal(info.HTTPProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.NoProxy, "example.com")) _, err := c.ImagePull(ctx, "example.org:5000/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") @@ -199,12 +205,6 @@ func TestDaemonProxy(t *testing.T) { _, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5000", "should not have used proxy") - - info := d.Info(t) - assert.Equal(t, info.HTTPProxy, proxyServer.URL) - assert.Equal(t, info.HTTPSProxy, proxyServer.URL) - assert.Equal(t, info.NoProxy, "example.com") - d.Stop(t) }) // Configure proxy through command-line flags @@ -218,6 +218,7 @@ func TestDaemonProxy(t *testing.T) { d := daemon.New(t) d.Start(t, "--http-proxy", proxyServer.URL, "--https-proxy", proxyServer.URL, "--no-proxy", "example.com") + defer d.Stop(t) logs, err := d.ReadLogFile() assert.NilError(t, err) @@ -231,6 +232,11 @@ func TestDaemonProxy(t *testing.T) { defer func() { _ = c.Close() }() ctx := context.Background() + info := d.Info(t) + assert.Check(t, is.Equal(info.HTTPProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.NoProxy, "example.com")) + _, err = c.ImagePull(ctx, "example.org:5001/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5001") @@ -239,13 +245,6 @@ func TestDaemonProxy(t *testing.T) { _, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5001", "should not have used proxy") - - info := d.Info(t) - assert.Equal(t, info.HTTPProxy, proxyServer.URL) - assert.Equal(t, info.HTTPSProxy, proxyServer.URL) - assert.Equal(t, info.NoProxy, "example.com") - - d.Stop(t) }) // Configure proxy through configuration file @@ -267,6 +266,12 @@ func TestDaemonProxy(t *testing.T) { assert.NilError(t, os.WriteFile(configFile, []byte(configJSON), 0644)) d.Start(t, "--config-file", configFile) + defer d.Stop(t) + + info := d.Info(t) + assert.Check(t, is.Equal(info.HTTPProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) + assert.Check(t, is.Equal(info.NoProxy, "example.com")) logs, err := d.ReadLogFile() assert.NilError(t, err) @@ -285,11 +290,6 @@ func TestDaemonProxy(t *testing.T) { assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5002", "should not have used proxy") - info := d.Info(t) - assert.Equal(t, info.HTTPProxy, proxyServer.URL) - assert.Equal(t, info.HTTPSProxy, proxyServer.URL) - assert.Equal(t, info.NoProxy, "example.com") - d.Stop(t) }) From a49bca97dfffb0f9bf42986721d401d58c2f76ae Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 28 Jul 2023 21:21:17 +0000 Subject: [PATCH 014/105] Fix daemon proxy test for "reload sanitized" I noticed this was always being skipped because of race conditions checking the logs. This change adds a log scanner which will look through the logs line by line rather than allocating a big buffer. Additionally it adds a `poll.Check` which we can use to actually wait for the desired log entry. Signed-off-by: Brian Goff Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 476e788090feafb246f800e71e926aa76d671fb3) Signed-off-by: Sebastiaan van Stijn --- integration/daemon/daemon_test.go | 25 ++++++++----------- testutil/daemon/daemon.go | 40 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index 28c5c422cd41e..78d13d1256c23 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -22,6 +22,7 @@ import ( "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" "gotest.tools/v3/icmd" + "gotest.tools/v3/poll" "gotest.tools/v3/skip" ) @@ -319,32 +320,26 @@ func TestDaemonProxy(t *testing.T) { // Make sure values are sanitized when reloading the daemon-config t.Run("reload sanitized", func(t *testing.T) { + t.Parallel() + + ctx := context.Background() const ( proxyRawURL = "https://" + userPass + "example.org" proxyURL = "https://xxxxx:xxxxx@example.org" ) d := daemon.New(t) - d.Start(t, "--http-proxy", proxyRawURL, "--https-proxy", proxyRawURL, "--no-proxy", "example.com") + d.Start(t, "--iptables=false", "--http-proxy", proxyRawURL, "--https-proxy", proxyRawURL, "--no-proxy", "example.com") defer d.Stop(t) err := d.Signal(syscall.SIGHUP) assert.NilError(t, err) - logs, err := d.ReadLogFile() - assert.NilError(t, err) + poll.WaitOn(t, d.PollCheckLogs(ctx, "Reloaded configuration:")) + poll.WaitOn(t, d.PollCheckLogs(ctx, proxyURL)) - // FIXME: there appears to ba a race condition, which causes ReadLogFile - // to not contain the full logs after signaling the daemon to reload, - // causing the test to fail here. As a workaround, check if we - // received the "reloaded" message after signaling, and only then - // check that it's sanitized properly. For more details on this - // issue, see https://github.com/moby/moby/pull/42835/files#r713120315 - if !strings.Contains(string(logs), "Reloaded configuration:") { - t.Skip("Skipping test, because we did not find 'Reloaded configuration' in the logs") - } - - assert.Assert(t, is.Contains(string(logs), proxyURL)) - assert.Assert(t, !strings.Contains(string(logs), userPass), "logs should not contain the non-sanitized proxy URL: %s", string(logs)) + ok, logs, err := d.ScanLogs(ctx, userPass) + assert.NilError(t, err) + assert.Assert(t, !ok, "logs should not contain the non-sanitized proxy URL: %s", logs) }) } diff --git a/testutil/daemon/daemon.go b/testutil/daemon/daemon.go index f0b5537b3089d..67f7c9297a4d2 100644 --- a/testutil/daemon/daemon.go +++ b/testutil/daemon/daemon.go @@ -1,8 +1,10 @@ package daemon // import "github.com/docker/docker/testutil/daemon" import ( + "bufio" "context" "encoding/json" + "io" "net/http" "os" "os/exec" @@ -25,6 +27,7 @@ import ( "github.com/docker/go-connections/tlsconfig" "github.com/pkg/errors" "gotest.tools/v3/assert" + "gotest.tools/v3/poll" ) // LogT is the subset of the testing.TB interface used by the daemon. @@ -310,6 +313,43 @@ func (d *Daemon) TailLogsT(t LogT, n int) { } } +// PollCheckLogs is a poll.Check that checks the daemon logs for the passed in string (`contains`). +func (d *Daemon) PollCheckLogs(ctx context.Context, contains string) poll.Check { + return func(t poll.LogT) poll.Result { + ok, _, err := d.ScanLogs(ctx, contains) + if err != nil { + return poll.Error(err) + } + if !ok { + return poll.Continue("waiting for %q in daemon logs", contains) + } + return poll.Success() + } +} + +// ScanLogs scans the daemon logs for the passed in string (`contains`). +// If the context is canceled, the function returns false but does not error out the test. +func (d *Daemon) ScanLogs(ctx context.Context, contains string) (bool, string, error) { + stat, err := d.logFile.Stat() + if err != nil { + return false, "", err + } + rdr := io.NewSectionReader(d.logFile, 0, stat.Size()) + + scanner := bufio.NewScanner(rdr) + for scanner.Scan() { + if strings.Contains(scanner.Text(), contains) { + return true, scanner.Text(), nil + } + select { + case <-ctx.Done(): + return false, "", ctx.Err() + default: + } + } + return false, "", scanner.Err() +} + // TailLogs tails N lines from the daemon logs func (d *Daemon) TailLogs(n int) ([][]byte, error) { logF, err := os.Open(d.logFile.Name()) From 4cd50eb1edc2032dc1920c2615a2ce2a80d4bf7d Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 28 Jul 2023 22:15:10 +0000 Subject: [PATCH 015/105] TestDaemonProxy: use new scanners to check logs Also fixes up some cleanup issues. Signed-off-by: Brian Goff Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 1a51898d2ea45a8f6e0d349878ea410782a24208) Signed-off-by: Sebastiaan van Stijn --- integration/daemon/daemon_test.go | 142 +++++++++++++++++------------- testutil/daemon/daemon.go | 46 ++++++++-- 2 files changed, 117 insertions(+), 71 deletions(-) diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index 78d13d1256c23..f4565c9616766 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -9,7 +9,6 @@ import ( "os/exec" "path/filepath" "runtime" - "strings" "syscall" "testing" @@ -170,27 +169,34 @@ func TestDaemonProxy(t *testing.T) { skip.If(t, runtime.GOOS == "windows", "cannot start multiple daemons on windows") skip.If(t, os.Getenv("DOCKER_ROOTLESS") != "", "cannot connect to localhost proxy in rootless environment") - var received string - proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - received = r.Host - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte("OK")) - })) - defer proxyServer.Close() + newProxy := func(rcvd *string, t *testing.T) *httptest.Server { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *rcvd = r.Host + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte("OK")) + })) + t.Cleanup(s.Close) + return s + } const userPass = "myuser:mypassword@" // Configure proxy through env-vars t.Run("environment variables", func(t *testing.T) { - t.Setenv("HTTP_PROXY", proxyServer.URL) - t.Setenv("HTTPS_PROXY", proxyServer.URL) - t.Setenv("NO_PROXY", "example.com") + t.Parallel() - d := daemon.New(t) - c := d.NewClientT(t) - defer func() { _ = c.Close() }() ctx := context.Background() - d.Start(t) + var received string + proxyServer := newProxy(&received, t) + + d := daemon.New(t, daemon.WithEnvVars( + "HTTP_PROXY="+proxyServer.URL, + "HTTPS_PROXY="+proxyServer.URL, + "NO_PROXY=example.com", + )) + c := d.NewClientT(t) + + d.Start(t, "--iptables=false") defer d.Stop(t) info := d.Info(t) @@ -210,35 +216,45 @@ func TestDaemonProxy(t *testing.T) { // Configure proxy through command-line flags t.Run("command-line options", func(t *testing.T) { - t.Setenv("HTTP_PROXY", "http://"+userPass+"from-env-http.invalid") - t.Setenv("http_proxy", "http://"+userPass+"from-env-http.invalid") - t.Setenv("HTTPS_PROXY", "https://"+userPass+"myuser:mypassword@from-env-https.invalid") - t.Setenv("https_proxy", "https://"+userPass+"myuser:mypassword@from-env-https.invalid") - t.Setenv("NO_PROXY", "ignore.invalid") - t.Setenv("no_proxy", "ignore.invalid") + t.Parallel() - d := daemon.New(t) - d.Start(t, "--http-proxy", proxyServer.URL, "--https-proxy", proxyServer.URL, "--no-proxy", "example.com") + ctx := context.Background() + var received string + proxyServer := newProxy(&received, t) + + d := daemon.New(t, daemon.WithEnvVars( + "HTTP_PROXY="+"http://"+userPass+"from-env-http.invalid", + "http_proxy="+"http://"+userPass+"from-env-http.invalid", + "HTTPS_PROXY="+"https://"+userPass+"myuser:mypassword@from-env-https-invalid", + "https_proxy="+"https://"+userPass+"myuser:mypassword@from-env-https-invalid", + "NO_PROXY=ignore.invalid", + "no_proxy=ignore.invalid", + )) + d.Start(t, "--iptables=false", "--http-proxy", proxyServer.URL, "--https-proxy", proxyServer.URL, "--no-proxy", "example.com") defer d.Stop(t) - logs, err := d.ReadLogFile() - assert.NilError(t, err) - assert.Assert(t, is.Contains(string(logs), "overriding existing proxy variable with value from configuration")) - for _, v := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} { - assert.Assert(t, is.Contains(string(logs), "name="+v)) - assert.Assert(t, !strings.Contains(string(logs), userPass), "logs should not contain the non-sanitized proxy URL: %s", string(logs)) - } - c := d.NewClientT(t) - defer func() { _ = c.Close() }() - ctx := context.Background() info := d.Info(t) assert.Check(t, is.Equal(info.HTTPProxy, proxyServer.URL)) assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) assert.Check(t, is.Equal(info.NoProxy, "example.com")) - _, err = c.ImagePull(ctx, "example.org:5001/some/image:latest", types.ImagePullOptions{}) + ok, _ := d.ScanLogsT(ctx, t, daemon.ScanLogsMatchAll( + "overriding existing proxy variable with value from configuration", + "http_proxy", + "HTTP_PROXY", + "https_proxy", + "HTTPS_PROXY", + "no_proxy", + "NO_PROXY", + )) + assert.Assert(t, ok) + + ok, logs := d.ScanLogsT(ctx, t, daemon.ScanLogsMatchString(userPass)) + assert.Assert(t, !ok, "logs should not contain the non-sanitized proxy URL: %s", logs) + + _, err := c.ImagePull(ctx, "example.org:5001/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5001") @@ -250,23 +266,27 @@ func TestDaemonProxy(t *testing.T) { // Configure proxy through configuration file t.Run("configuration file", func(t *testing.T) { - t.Setenv("HTTP_PROXY", "http://"+userPass+"from-env-http.invalid") - t.Setenv("http_proxy", "http://"+userPass+"from-env-http.invalid") - t.Setenv("HTTPS_PROXY", "https://"+userPass+"myuser:mypassword@from-env-https.invalid") - t.Setenv("https_proxy", "https://"+userPass+"myuser:mypassword@from-env-https.invalid") - t.Setenv("NO_PROXY", "ignore.invalid") - t.Setenv("no_proxy", "ignore.invalid") + t.Parallel() + ctx := context.Background() - d := daemon.New(t) + var received string + proxyServer := newProxy(&received, t) + + d := daemon.New(t, daemon.WithEnvVars( + "HTTP_PROXY="+"http://"+userPass+"from-env-http.invalid", + "http_proxy="+"http://"+userPass+"from-env-http.invalid", + "HTTPS_PROXY="+"https://"+userPass+"myuser:mypassword@from-env-https-invalid", + "https_proxy="+"https://"+userPass+"myuser:mypassword@from-env-https-invalid", + "NO_PROXY=ignore.invalid", + "no_proxy=ignore.invalid", + )) c := d.NewClientT(t) - defer func() { _ = c.Close() }() - ctx := context.Background() configFile := filepath.Join(d.RootDir(), "daemon.json") configJSON := fmt.Sprintf(`{"proxies":{"http-proxy":%[1]q, "https-proxy": %[1]q, "no-proxy": "example.com"}}`, proxyServer.URL) assert.NilError(t, os.WriteFile(configFile, []byte(configJSON), 0644)) - d.Start(t, "--config-file", configFile) + d.Start(t, "--iptables=false", "--config-file", configFile) defer d.Stop(t) info := d.Info(t) @@ -274,15 +294,17 @@ func TestDaemonProxy(t *testing.T) { assert.Check(t, is.Equal(info.HTTPSProxy, proxyServer.URL)) assert.Check(t, is.Equal(info.NoProxy, "example.com")) - logs, err := d.ReadLogFile() - assert.NilError(t, err) - assert.Assert(t, is.Contains(string(logs), "overriding existing proxy variable with value from configuration")) - for _, v := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} { - assert.Assert(t, is.Contains(string(logs), "name="+v)) - assert.Assert(t, !strings.Contains(string(logs), userPass), "logs should not contain the non-sanitized proxy URL: %s", string(logs)) - } - - _, err = c.ImagePull(ctx, "example.org:5002/some/image:latest", types.ImagePullOptions{}) + d.ScanLogsT(ctx, t, daemon.ScanLogsMatchAll( + "overriding existing proxy variable with value from configuration", + "http_proxy", + "HTTP_PROXY", + "https_proxy", + "HTTPS_PROXY", + "no_proxy", + "NO_PROXY", + )) + + _, err := c.ImagePull(ctx, "example.org:5002/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5002") @@ -290,12 +312,11 @@ func TestDaemonProxy(t *testing.T) { _, err = c.ImagePull(ctx, "example.com/some/image:latest", types.ImagePullOptions{}) assert.ErrorContains(t, err, "", "pulling should have failed") assert.Equal(t, received, "example.org:5002", "should not have used proxy") - - d.Stop(t) }) // Conflicting options (passed both through command-line options and config file) t.Run("conflicting options", func(t *testing.T) { + ctx := context.Background() const ( proxyRawURL = "https://" + userPass + "example.org" proxyURL = "https://xxxxx:xxxxx@example.org" @@ -309,13 +330,12 @@ func TestDaemonProxy(t *testing.T) { err := d.StartWithError("--http-proxy", proxyRawURL, "--https-proxy", proxyRawURL, "--no-proxy", "example.com", "--config-file", configFile, "--validate") assert.ErrorContains(t, err, "daemon exited during startup") - logs, err := d.ReadLogFile() - assert.NilError(t, err) + expected := fmt.Sprintf( `the following directives are specified both as a flag and in the configuration file: http-proxy: (from flag: %[1]s, from file: %[1]s), https-proxy: (from flag: %[1]s, from file: %[1]s), no-proxy: (from flag: example.com, from file: example.com)`, proxyURL, ) - assert.Assert(t, is.Contains(string(logs), expected)) + poll.WaitOn(t, d.PollCheckLogs(ctx, daemon.ScanLogsMatchString(expected))) }) // Make sure values are sanitized when reloading the daemon-config @@ -334,11 +354,9 @@ func TestDaemonProxy(t *testing.T) { err := d.Signal(syscall.SIGHUP) assert.NilError(t, err) - poll.WaitOn(t, d.PollCheckLogs(ctx, "Reloaded configuration:")) - poll.WaitOn(t, d.PollCheckLogs(ctx, proxyURL)) + poll.WaitOn(t, d.PollCheckLogs(ctx, daemon.ScanLogsMatchAll("Reloaded configuration:", proxyURL))) - ok, logs, err := d.ScanLogs(ctx, userPass) - assert.NilError(t, err) + ok, logs := d.ScanLogsT(ctx, t, daemon.ScanLogsMatchString(userPass)) assert.Assert(t, !ok, "logs should not contain the non-sanitized proxy URL: %s", logs) }) } diff --git a/testutil/daemon/daemon.go b/testutil/daemon/daemon.go index 67f7c9297a4d2..360c39c89d592 100644 --- a/testutil/daemon/daemon.go +++ b/testutil/daemon/daemon.go @@ -277,6 +277,7 @@ func (d *Daemon) NewClientT(t testing.TB, extraOpts ...client.Opt) *client.Clien c, err := d.NewClient(extraOpts...) assert.NilError(t, err, "[%s] could not create daemon client", d.id) + t.Cleanup(func() { c.Close() }) return c } @@ -313,23 +314,51 @@ func (d *Daemon) TailLogsT(t LogT, n int) { } } -// PollCheckLogs is a poll.Check that checks the daemon logs for the passed in string (`contains`). -func (d *Daemon) PollCheckLogs(ctx context.Context, contains string) poll.Check { +// PollCheckLogs is a poll.Check that checks the daemon logs using the passed in match function. +func (d *Daemon) PollCheckLogs(ctx context.Context, match func(s string) bool) poll.Check { return func(t poll.LogT) poll.Result { - ok, _, err := d.ScanLogs(ctx, contains) + ok, _, err := d.ScanLogs(ctx, match) if err != nil { return poll.Error(err) } if !ok { - return poll.Continue("waiting for %q in daemon logs", contains) + return poll.Continue("waiting for daemon logs match") } return poll.Success() } } -// ScanLogs scans the daemon logs for the passed in string (`contains`). -// If the context is canceled, the function returns false but does not error out the test. -func (d *Daemon) ScanLogs(ctx context.Context, contains string) (bool, string, error) { +// ScanLogsMatchString returns a function that can be used to scan the daemon logs for the passed in string (`contains`). +func ScanLogsMatchString(contains string) func(string) bool { + return func(line string) bool { + return strings.Contains(line, contains) + } +} + +// ScanLogsMatchAll returns a function that can be used to scan the daemon logs until *all* the passed in strings are matched +func ScanLogsMatchAll(contains ...string) func(string) bool { + matched := make(map[string]bool) + return func(line string) bool { + for _, c := range contains { + if strings.Contains(line, c) { + matched[c] = true + } + } + return len(matched) == len(contains) + } +} + +// ScanLogsT uses `ScanLogs` to match the daemon logs using the passed in match function. +// If there is an error or the match fails, the test will fail. +func (d *Daemon) ScanLogsT(ctx context.Context, t testing.TB, match func(s string) bool) (bool, string) { + t.Helper() + ok, line, err := d.ScanLogs(ctx, match) + assert.NilError(t, err) + return ok, line +} + +// ScanLogs scans the daemon logs and passes each line to the match function. +func (d *Daemon) ScanLogs(ctx context.Context, match func(s string) bool) (bool, string, error) { stat, err := d.logFile.Stat() if err != nil { return false, "", err @@ -338,7 +367,7 @@ func (d *Daemon) ScanLogs(ctx context.Context, contains string) (bool, string, e scanner := bufio.NewScanner(rdr) for scanner.Scan() { - if strings.Contains(scanner.Text(), contains) { + if match(scanner.Text()) { return true, scanner.Text(), nil } select { @@ -364,7 +393,6 @@ func (d *Daemon) TailLogs(n int) ([][]byte, error) { } return lines, nil - } // Start starts the daemon and return once it is ready to receive requests. From b7c5385b81b5a7a58f6a2cc0d78d380f1fb6931c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 1 Aug 2023 23:46:01 +0200 Subject: [PATCH 016/105] update to go1.20.7 Includes a fix for CVE-2023-29409 go1.20.7 (released 2023-08-01) includes a security fix to the crypto/tls package, as well as bug fixes to the assembler and the compiler. See the Go 1.20.7 milestone on our issue tracker for details: - https://github.com/golang/go/issues?q=milestone%3AGo1.20.7+label%3ACherryPickApproved - full diff: https://github.com/golang/go/compare/go1.20.6...go1.20.7 From the mailing list announcement: [security] Go 1.20.7 and Go 1.19.12 are released Hello gophers, We have just released Go versions 1.20.7 and 1.19.12, minor point releases. These minor releases include 1 security fixes following the security policy: - crypto/tls: restrict RSA keys in certificates to <= 8192 bits Extremely large RSA keys in certificate chains can cause a client/server to expend significant CPU time verifying signatures. Limit this by restricting the size of RSA keys transmitted during handshakes to <= 8192 bits. Based on a survey of publicly trusted RSA keys, there are currently only three certificates in circulation with keys larger than this, and all three appear to be test certificates that are not actively deployed. It is possible there are larger keys in use in private PKIs, but we target the web PKI, so causing breakage here in the interests of increasing the default safety of users of crypto/tls seems reasonable. Thanks to Mateusz Poliwczak for reporting this issue. View the release notes for more information: https://go.dev/doc/devel/release#go1.20.7 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit d5cb7cdeae32f071dfa243c2a34925a23dd50679) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/.windows.yml | 2 +- .github/workflows/buildkit.yml | 2 +- .github/workflows/test.yml | 2 +- Dockerfile | 2 +- Dockerfile.simple | 2 +- Dockerfile.windows | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml index 66812324a817d..5e8736e34de9b 100644 --- a/.github/workflows/.windows.yml +++ b/.github/workflows/.windows.yml @@ -15,7 +15,7 @@ on: default: false env: - GO_VERSION: "1.20.6" + GO_VERSION: "1.20.7" GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 WINDOWS_BASE_IMAGE: mcr.microsoft.com/windows/servercore diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 7de6e0c22f941..95910dbf3c9e2 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -13,7 +13,7 @@ on: pull_request: env: - GO_VERSION: "1.20.6" + GO_VERSION: "1.20.7" DESTDIR: ./build jobs: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d5fb2f6a8c5c..2e7467f418c75 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ on: pull_request: env: - GO_VERSION: "1.20.6" + GO_VERSION: "1.20.7" GOTESTLIST_VERSION: v0.3.1 TESTSTAT_VERSION: v0.1.3 ITG_CLI_MATRIX_SIZE: 6 diff --git a/Dockerfile b/Dockerfile index 9a8e5ea7ffaaa..d4cbee5021570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20.6 +ARG GO_VERSION=1.20.7 ARG BASE_DEBIAN_DISTRO="bullseye" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" ARG XX_VERSION=1.2.1 diff --git a/Dockerfile.simple b/Dockerfile.simple index 8605aa3fcd0c7..91d2eaf04d068 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -5,7 +5,7 @@ # This represents the bare minimum required to build and test Docker. -ARG GO_VERSION=1.20.6 +ARG GO_VERSION=1.20.7 ARG BASE_DEBIAN_DISTRO="bullseye" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" diff --git a/Dockerfile.windows b/Dockerfile.windows index e98f714e16c75..d3783a9331bf7 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -165,7 +165,7 @@ FROM microsoft/windowsservercore # Use PowerShell as the default shell SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -ARG GO_VERSION=1.20.6 +ARG GO_VERSION=1.20.7 ARG GOTESTSUM_VERSION=v1.8.2 ARG GOWINRES_VERSION=v0.3.0 ARG CONTAINERD_VERSION=v1.7.3 From 128838227e0cf234e2a1b4f4610189dea363d120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Tue, 8 Aug 2023 12:52:28 +0200 Subject: [PATCH 017/105] hack/test: Don't fail-fast before integration-cli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If TEST_INTEGRATION_FAIL_FAST is not set, run the integration-cli tests even if integration tests failed. Signed-off-by: Paweł Gronowski (cherry picked from commit 6841a53d1764bd65d08fe655a147ae3c24c977c1) Signed-off-by: Paweł Gronowski --- hack/make/.integration-test-helpers | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index 177a2ec8808be..e9062ac66cdec 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -55,11 +55,23 @@ fi run_test_integration() { set_platform_timeout + local failed=0 if [ -z "${TEST_SKIP_INTEGRATION}" ]; then - run_test_integration_suites "${integration_api_dirs}" + if ! run_test_integration_suites "${integration_api_dirs}"; then + if [ -n "${TEST_INTEGRATION_FAIL_FAST}" ]; then + return 1 + fi + failed=1 + fi fi if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then - TIMEOUT=360m run_test_integration_suites integration-cli + if ! TIMEOUT=360m run_test_integration_suites integration-cli; then + return 1 + fi + fi + + if [ $failed -eq 1 ]; then + return 1 fi } @@ -99,13 +111,13 @@ run_test_integration_suites() { -- go tool test2json -p "${pkgname}" -t ./test.main ${pkgtestflags} ); then if [ -n "${TEST_INTEGRATION_FAIL_FAST}" ]; then - exit 1 + return 1 fi failed=1 fi done if [ $failed -eq 1 ]; then - exit 1 + return 1 fi } From 749e687e1b52e87a51a92df02334603a68072462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Tue, 25 Apr 2023 12:00:12 +0200 Subject: [PATCH 018/105] integration/windows: Unskip some kill tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unskip: - TestKillWithStopSignalAndRestartPolicies - TestKillStoppedContainer integration tests on Windows. Signed-off-by: Paweł Gronowski (cherry picked from commit dd1c95edcde99cc5a2673567c4c47e939b8a4e41) Signed-off-by: Paweł Gronowski --- integration/container/kill_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integration/container/kill_test.go b/integration/container/kill_test.go index 42adb7867a3d0..219cd55bb2fb1 100644 --- a/integration/container/kill_test.go +++ b/integration/container/kill_test.go @@ -82,7 +82,6 @@ func TestKillContainer(t *testing.T) { } func TestKillWithStopSignalAndRestartPolicies(t *testing.T) { - skip.If(t, testEnv.OSType == "windows", "Windows only supports 1.25 or later") defer setupTest(t)() client := testEnv.APIClient() @@ -103,6 +102,11 @@ func TestKillWithStopSignalAndRestartPolicies(t *testing.T) { }, } + var pollOpts []poll.SettingOp + if runtime.GOOS == "windows" { + pollOpts = append(pollOpts, poll.WithTimeout(StopContainerWindowsPollTimeout)) + } + for _, tc := range testCases { tc := tc t.Run(tc.doc, func(t *testing.T) { @@ -115,13 +119,12 @@ func TestKillWithStopSignalAndRestartPolicies(t *testing.T) { err := client.ContainerKill(ctx, id, "TERM") assert.NilError(t, err) - poll.WaitOn(t, container.IsInState(ctx, client, id, tc.status), poll.WithDelay(100*time.Millisecond)) + poll.WaitOn(t, container.IsInState(ctx, client, id, tc.status), pollOpts...) }) } } func TestKillStoppedContainer(t *testing.T) { - skip.If(t, testEnv.OSType == "windows", "Windows only supports 1.25 or later") defer setupTest(t)() ctx := context.Background() client := testEnv.APIClient() From b7d1e98ae73c11b449acc5065f8315f42d65a812 Mon Sep 17 00:00:00 2001 From: Albin Kerouanton Date: Tue, 30 May 2023 14:23:02 +0200 Subject: [PATCH 019/105] libnet/d/bridge: Allow IPv6 ICC from any IP address IPv6 ipt rules are exactly the same as IPv4 rules, although both protocol don't use the same networking model. This has bad consequences, for instance: 1. the current v6 rules disallow Neighbor Solication/Advertisement ; 2. multicast addresses can't be used ; 3. link-local addresses are blocked too. To solve this, this commit changes the following rules: ``` -A DOCKER-ISOLATION-STAGE-1 ! -s fdf1:a844:380c:b247::/64 -o br-21502e5b2c6c -j DROP -A DOCKER-ISOLATION-STAGE-1 ! -d fdf1:a844:380c:b247::/64 -i br-21502e5b2c6c -j DROP ``` into: ``` -A DOCKER-ISOLATION-STAGE-1 ! -s fdf1:a844:380c:b247::/64 ! -i br-21502e5b2c6c -o br-21502e5b2c6c -j DROP -A DOCKER-ISOLATION-STAGE-1 ! -d fdf1:a844:380c:b247::/64 -i br-21502e5b2c6c ! -o br-21502e5b2c6c -j DROP ``` These rules only limit the traffic ingressing/egressing the bridge, but not traffic between veth on the same bridge. Note that, the Kernel takes care of dropping invalid IPv6 packets, eg. loopback spoofing, thus these rules don't need to be more specific. Solve #45460. Signed-off-by: Albin Kerouanton (cherry picked from commit da9e44a620db39307f5548a1451be9c434c5b34d) Signed-off-by: Sebastiaan van Stijn --- libnetwork/drivers/bridge/setup_ip_tables.go | 23 +++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/libnetwork/drivers/bridge/setup_ip_tables.go b/libnetwork/drivers/bridge/setup_ip_tables.go index bd2822e39cab9..f53d0a93ae988 100644 --- a/libnetwork/drivers/bridge/setup_ip_tables.go +++ b/libnetwork/drivers/bridge/setup_ip_tables.go @@ -397,15 +397,21 @@ func removeIPChains(version iptables.IPVersion) { } func setupInternalNetworkRules(bridgeIface string, addr *net.IPNet, icc, insert bool) error { - var ( - inDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}} - outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}} - ) - - version := iptables.IPv4 - - if addr.IP.To4() == nil { + var version iptables.IPVersion + var inDropRule, outDropRule iptRule + + if addr.IP.To4() != nil { + version = iptables.IPv4 + inDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{ + "-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}} + outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{ + "-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}} + } else { version = iptables.IPv6 + inDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{ + "-i", bridgeIface, "!", "-o", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}} + outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{ + "!", "-i", bridgeIface, "-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}} } if err := programChainRule(version, inDropRule, "DROP INCOMING", insert); err != nil { @@ -414,6 +420,7 @@ func setupInternalNetworkRules(bridgeIface string, addr *net.IPNet, icc, insert if err := programChainRule(version, outDropRule, "DROP OUTGOING", insert); err != nil { return err } + // Set Inter Container Communication. return setIcc(version, bridgeIface, icc, insert) } From 25b709df48addea1b25ca92189078c204f214303 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 27 Jul 2023 16:20:49 +0200 Subject: [PATCH 020/105] windows: fix --register-service when executed from within binary directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Go 1.15.7 contained a security fix for CVE-2021-3115, which allowed arbitrary code to be executed at build time when using cgo on Windows. This issue was not limited to the go command itself, and could also affect binaries that use `os.Command`, `os.LookPath`, etc. From the related blogpost (https://blog.golang.org/path-security): > Are your own programs affected? > > If you use exec.LookPath or exec.Command in your own programs, you only need to > be concerned if you (or your users) run your program in a directory with untrusted > contents. If so, then a subprocess could be started using an executable from dot > instead of from a system directory. (Again, using an executable from dot happens > always on Windows and only with uncommon PATH settings on Unix.) > > If you are concerned, then we’ve published the more restricted variant of os/exec > as golang.org/x/sys/execabs. You can use it in your program by simply replacing At time of the go1.15 release, the Go team considered changing the behavior of `os.LookPath()` and `exec.LookPath()` to be a breaking change, and made the behavior "opt-in" by providing the `golang.org/x/sys/execabs` package as a replacement. However, for the go1.19 release, this changed, and the default behavior of `os.LookPath()` and `exec.LookPath()` was changed. From the release notes: https://go.dev/doc/go1.19#os-exec-path > Command and LookPath no longer allow results from a PATH search to be found > relative to the current directory. This removes a common source of security > problems but may also break existing programs that depend on using, say, > exec.Command("prog") to run a binary named prog (or, on Windows, prog.exe) > in the current directory. See the os/exec package documentation for information > about how best to update such programs. > > On Windows, Command and LookPath now respect the NoDefaultCurrentDirectoryInExePath > environment variable, making it possible to disable the default implicit search > of “.” in PATH lookups on Windows systems. A result of this change was that registering the daemon as a Windows service no longer worked when done from within the directory of the binary itself: C:\> cd "Program Files\Docker\Docker\resources" C:\Program Files\Docker\Docker\resources> dockerd --register-service exec: "dockerd": cannot run executable found relative to current directory Note that using an absolute path would work around the issue: C:\Program Files\Docker\Docker>resources\dockerd.exe --register-service This patch changes `registerService()` to use `os.Executable()`, instead of depending on `os.Args[0]` and `exec.LookPath()` for resolving the absolute path of the binary. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 3e8fda0a709bb7e706547d7b656e380eba965995) Signed-off-by: Sebastiaan van Stijn --- cmd/dockerd/service_windows.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/cmd/dockerd/service_windows.go b/cmd/dockerd/service_windows.go index 4510d79f88db8..cb7502d447a45 100644 --- a/cmd/dockerd/service_windows.go +++ b/cmd/dockerd/service_windows.go @@ -7,7 +7,6 @@ import ( "io" "log" "os" - "os/exec" "path/filepath" "time" @@ -145,16 +144,8 @@ func (h *etwHook) Fire(e *logrus.Entry) error { return windows.ReportEvent(h.log.Handle, etype, 0, eid, 0, count, 0, &ss[0], nil) } -func getServicePath() (string, error) { - p, err := exec.LookPath(os.Args[0]) - if err != nil { - return "", err - } - return filepath.Abs(p) -} - func registerService() error { - p, err := getServicePath() + p, err := os.Executable() if err != nil { return err } From 5f0df8c53429c03e9746bbbf1355d106f7f8e748 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 13 Aug 2023 22:08:44 +0200 Subject: [PATCH 021/105] vendor github.com/containerd/ttrpc v1.1.2 full diff: https://github.com/containerd/ttrpc/compare/v1.1.1...v1.1.2 Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 ++-- vendor/github.com/containerd/ttrpc/server.go | 2 +- vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vendor.mod b/vendor.mod index 93b49543cb836..8212b6aac36fb 100644 --- a/vendor.mod +++ b/vendor.mod @@ -121,7 +121,7 @@ require ( github.com/containerd/go-runc v1.1.0 // indirect github.com/containerd/nydus-snapshotter v0.3.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.13.0 // indirect - github.com/containerd/ttrpc v1.1.1 // indirect + github.com/containerd/ttrpc v1.1.2 // indirect github.com/containerd/typeurl v1.0.2 // indirect github.com/containernetworking/cni v1.1.1 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect diff --git a/vendor.sum b/vendor.sum index 7a265df2c3284..a27f32c1a2b78 100644 --- a/vendor.sum +++ b/vendor.sum @@ -419,8 +419,8 @@ github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0x github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/ttrpc v1.1.1 h1:NoRHS/z8UiHhpY1w0xcOqoJDGf2DHyzXrF0H4l5AE8c= -github.com/containerd/ttrpc v1.1.1/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/ttrpc v1.1.2 h1:4jH6OQDQqjfVD2b5TJS5TxmGuLGmp5WW7KtW2TWOP7c= +github.com/containerd/ttrpc v1.1.2/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= diff --git a/vendor/github.com/containerd/ttrpc/server.go b/vendor/github.com/containerd/ttrpc/server.go index e4c07b60fb8ce..5c62d169f3627 100644 --- a/vendor/github.com/containerd/ttrpc/server.go +++ b/vendor/github.com/containerd/ttrpc/server.go @@ -468,7 +468,7 @@ func (c *serverConn) run(sctx context.Context) { // branch. Basically, it means that we are no longer receiving // requests due to a terminal error. recvErr = nil // connection is now "closing" - if err == io.EOF || err == io.ErrUnexpectedEOF || errors.Is(err, syscall.ECONNRESET) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) { // The client went away and we should stop processing // requests, so that the client connection is closed return diff --git a/vendor/modules.txt b/vendor/modules.txt index 65ad85f86de6e..4baa79bd2ac05 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -323,7 +323,7 @@ github.com/containerd/nydus-snapshotter/pkg/errdefs ## explicit; go 1.16 github.com/containerd/stargz-snapshotter/estargz github.com/containerd/stargz-snapshotter/estargz/errorutil -# github.com/containerd/ttrpc v1.1.1 +# github.com/containerd/ttrpc v1.1.2 ## explicit; go 1.13 github.com/containerd/ttrpc # github.com/containerd/typeurl v1.0.2 From 7e7bc0f1bcb90c206ab04f958bc9b4c9f8349a34 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 29 Jul 2023 19:54:59 +0200 Subject: [PATCH 022/105] vendor: github.com/containerd/containerd v1.6.22 - full diff: https://github.com/containerd/containerd/compare/v1.6.21...v1.6.22 - release notes: https://github.com/containerd/containerd/releases/tag/v1.6.22 --- Notable Updates - RunC: Update runc binary to v1.1.8 - CRI: Fix `additionalGids`: it should fallback to `imageConfig.User` when `securityContext.RunAsUser`, `RunAsUsername` are empty - CRI: Write generated CNI config atomically - Fix concurrent writes for `UpdateContainerStats` - Make `checkContainerTimestamps` less strict on Windows - Port-Forward: Correctly handle known errors - Resolve `docker.NewResolver` race condition - SecComp: Always allow `name_to_handle_at` - Adding support to run hcsshim from local clone - Pinned image support - Runtime/V2/RunC: Handle early exits w/o big locks - CRITool: Move up to CRI-TOOLS v1.27.0 - Fix cpu architecture detection issue on emulated ARM platform - Task: Don't `close()` io before `cancel()` - Fix panic when remote differ returns empty result - Plugins: Notify readiness when registered plugins are ready - Unwrap io errors in server connection receive error handling Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 4d674897f3748a28347f2233237dbfae7c18cdc1) Signed-off-by: Sebastiaan van Stijn --- vendor.mod | 2 +- vendor.sum | 4 +- .../github.com/containerd/containerd/.mailmap | 2 + .../github.com/containerd/containerd/Makefile | 16 +- .../containerd/containerd/Vagrantfile | 2 +- .../github.com/containerd/containerd/diff.go | 3 + .../containerd/containerd/log/context.go | 51 ++++++ .../containerd/pkg/atomicfile/file.go | 148 ++++++++++++++++ .../containerd/platforms/cpuinfo.go | 98 +---------- .../containerd/platforms/cpuinfo_linux.go | 161 ++++++++++++++++++ .../containerd/platforms/cpuinfo_other.go | 60 +++++++ .../containerd/platforms/database.go | 7 - .../containerd/containerd/plugin/context.go | 13 +- .../containerd/remotes/docker/resolver.go | 25 ++- .../containerd/runtime/v2/shim/util.go | 20 +-- .../github.com/containerd/containerd/task.go | 11 +- .../containerd/containerd/version/version.go | 2 +- vendor/modules.txt | 5 +- 18 files changed, 492 insertions(+), 138 deletions(-) create mode 100644 vendor/github.com/containerd/containerd/pkg/atomicfile/file.go create mode 100644 vendor/github.com/containerd/containerd/platforms/cpuinfo_linux.go create mode 100644 vendor/github.com/containerd/containerd/platforms/cpuinfo_other.go diff --git a/vendor.mod b/vendor.mod index 8212b6aac36fb..7cda78ef2bd90 100644 --- a/vendor.mod +++ b/vendor.mod @@ -25,7 +25,7 @@ require ( github.com/bsphere/le_go v0.0.0-20200109081728-fc06dab2caa8 github.com/cloudflare/cfssl v0.0.0-20180323000720-5d63dbd981b5 github.com/containerd/cgroups/v3 v3.0.2 - github.com/containerd/containerd v1.6.21 + github.com/containerd/containerd v1.6.22 github.com/containerd/continuity v0.3.0 github.com/containerd/fifo v1.1.0 github.com/containerd/typeurl/v2 v2.1.0 diff --git a/vendor.sum b/vendor.sum index a27f32c1a2b78..04af7872226f0 100644 --- a/vendor.sum +++ b/vendor.sum @@ -370,8 +370,8 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.6.21 h1:eSTAmnvDKRPWan+MpSSfNyrtleXd86ogK9X8fMWpe/Q= -github.com/containerd/containerd v1.6.21/go.mod h1:apei1/i5Ux2FzrK6+DM/suEsGuK/MeVOfy8tR2q7Wnw= +github.com/containerd/containerd v1.6.22 h1:rGTIBxPJusM0evF6wKgIzuD+tV70nmx9eEjzHVm1JzI= +github.com/containerd/containerd v1.6.22/go.mod h1:BQAJdahvGz8xboAvxKg9hsDYIovn79Ea318anowQ1/o= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= diff --git a/vendor/github.com/containerd/containerd/.mailmap b/vendor/github.com/containerd/containerd/.mailmap index 11dcdc48c0883..3988d4797a90e 100644 --- a/vendor/github.com/containerd/containerd/.mailmap +++ b/vendor/github.com/containerd/containerd/.mailmap @@ -145,3 +145,5 @@ Zhoulin Xie Zhoulin Xie <42261994+JoeWrightss@users.noreply.github.com> zounengren 张潇 +Kazuyoshi Kato +Andrey Epifanov diff --git a/vendor/github.com/containerd/containerd/Makefile b/vendor/github.com/containerd/containerd/Makefile index 7441eeac66e12..f1b28ceb9c9f5 100644 --- a/vendor/github.com/containerd/containerd/Makefile +++ b/vendor/github.com/containerd/containerd/Makefile @@ -332,22 +332,26 @@ install-cri-deps: $(BINARIES) @$(INSTALL) $(BINARIES) $(CRIDIR)/bin endif +$(CRIDIR)/cri-containerd.DEPRECATED.txt: + @mkdir -p $(CRIDIR) + @$(INSTALL) -m 644 releases/cri-containerd.DEPRECATED.txt $@ + ifeq ($(GOOS),windows) -releases/$(CRIRELEASE).tar.gz: install-cri-deps +releases/$(CRIRELEASE).tar.gz: install-cri-deps $(CRIDIR)/cri-containerd.DEPRECATED.txt @echo "$(WHALE) $@" @cd $(CRIDIR) && tar -czf ../../releases/$(CRIRELEASE).tar.gz * -releases/$(CRICNIRELEASE).tar.gz: install-cri-deps +releases/$(CRICNIRELEASE).tar.gz: install-cri-deps $(CRIDIR)/cri-containerd.DEPRECATED.txt @echo "$(WHALE) $@" @cd $(CRIDIR) && tar -czf ../../releases/$(CRICNIRELEASE).tar.gz * else -releases/$(CRIRELEASE).tar.gz: install-cri-deps +releases/$(CRIRELEASE).tar.gz: install-cri-deps $(CRIDIR)/cri-containerd.DEPRECATED.txt @echo "$(WHALE) $@" - @tar -czf releases/$(CRIRELEASE).tar.gz -C $(CRIDIR) etc/crictl.yaml etc/systemd usr opt/containerd + @tar -czf releases/$(CRIRELEASE).tar.gz -C $(CRIDIR) cri-containerd.DEPRECATED.txt etc/crictl.yaml etc/systemd usr opt/containerd -releases/$(CRICNIRELEASE).tar.gz: install-cri-deps +releases/$(CRICNIRELEASE).tar.gz: install-cri-deps $(CRIDIR)/cri-containerd.DEPRECATED.txt @echo "$(WHALE) $@" - @tar -czf releases/$(CRICNIRELEASE).tar.gz -C $(CRIDIR) etc usr opt + @tar -czf releases/$(CRICNIRELEASE).tar.gz -C $(CRIDIR) cri-containerd.DEPRECATED.txt etc usr opt endif cri-release: releases/$(CRIRELEASE).tar.gz diff --git a/vendor/github.com/containerd/containerd/Vagrantfile b/vendor/github.com/containerd/containerd/Vagrantfile index f706788eccdda..95c3a359b41a5 100644 --- a/vendor/github.com/containerd/containerd/Vagrantfile +++ b/vendor/github.com/containerd/containerd/Vagrantfile @@ -93,7 +93,7 @@ EOF config.vm.provision "install-golang", type: "shell", run: "once" do |sh| sh.upload_path = "/tmp/vagrant-install-golang" sh.env = { - 'GO_VERSION': ENV['GO_VERSION'] || "1.19.9", + 'GO_VERSION': ENV['GO_VERSION'] || "1.19.11", } sh.inline = <<~SHELL #!/usr/bin/env bash diff --git a/vendor/github.com/containerd/containerd/diff.go b/vendor/github.com/containerd/containerd/diff.go index 445df019220c8..28012b1f02bc5 100644 --- a/vendor/github.com/containerd/containerd/diff.go +++ b/vendor/github.com/containerd/containerd/diff.go @@ -86,6 +86,9 @@ func (r *diffRemote) Compare(ctx context.Context, a, b []mount.Mount, opts ...di } func toDescriptor(d *types.Descriptor) ocispec.Descriptor { + if d == nil { + return ocispec.Descriptor{} + } return ocispec.Descriptor{ MediaType: d.MediaType, Digest: d.Digest, diff --git a/vendor/github.com/containerd/containerd/log/context.go b/vendor/github.com/containerd/containerd/log/context.go index 0db9562b82bba..b63c602f424a5 100644 --- a/vendor/github.com/containerd/containerd/log/context.go +++ b/vendor/github.com/containerd/containerd/log/context.go @@ -18,6 +18,7 @@ package log import ( "context" + "fmt" "github.com/sirupsen/logrus" ) @@ -35,6 +36,12 @@ var ( type ( loggerKey struct{} + + // Fields type to pass to `WithFields`, alias from `logrus`. + Fields = logrus.Fields + + // Level is a logging level + Level = logrus.Level ) const ( @@ -47,8 +54,52 @@ const ( // JSONFormat represents the JSON logging format JSONFormat = "json" + + // TraceLevel level. + TraceLevel = logrus.TraceLevel + + // DebugLevel level. + DebugLevel = logrus.DebugLevel + + // InfoLevel level. + InfoLevel = logrus.InfoLevel ) +// SetLevel sets log level globally. +func SetLevel(level string) error { + lvl, err := logrus.ParseLevel(level) + if err != nil { + return err + } + + logrus.SetLevel(lvl) + return nil +} + +// GetLevel returns the current log level. +func GetLevel() Level { + return logrus.GetLevel() +} + +// SetFormat sets log output format +func SetFormat(format string) error { + switch format { + case TextFormat: + logrus.SetFormatter(&logrus.TextFormatter{ + TimestampFormat: RFC3339NanoFixed, + FullTimestamp: true, + }) + case JSONFormat: + logrus.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: RFC3339NanoFixed, + }) + default: + return fmt.Errorf("unknown log format: %s", format) + } + + return nil +} + // WithLogger returns a new context with the provided logger. Use in // combination with logger.WithField(s) for great effect. func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context { diff --git a/vendor/github.com/containerd/containerd/pkg/atomicfile/file.go b/vendor/github.com/containerd/containerd/pkg/atomicfile/file.go new file mode 100644 index 0000000000000..7b870f7a787d7 --- /dev/null +++ b/vendor/github.com/containerd/containerd/pkg/atomicfile/file.go @@ -0,0 +1,148 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* +Package atomicfile provides a mechanism (on Unix-like platforms) to present a consistent view of a file to separate +processes even while the file is being written. This is accomplished by writing a temporary file, syncing to disk, and +renaming over the destination file name. + +Partial/inconsistent reads can occur due to: + 1. A process attempting to read the file while it is being written to (both in the case of a new file with a + short/incomplete write or in the case of an existing, updated file where new bytes may be written at the beginning + but old bytes may still be present after). + 2. Concurrent goroutines leading to multiple active writers of the same file. + +The above mechanism explicitly protects against (1) as all writes are to a file with a temporary name. + +There is no explicit protection against multiple, concurrent goroutines attempting to write the same file. However, +atomically writing the file should mean only one writer will "win" and a consistent file will be visible. + +Note: atomicfile is partially implemented for Windows. The Windows codepath performs the same operations, however +Windows does not guarantee that a rename operation is atomic; a crash in the middle may leave the destination file +truncated rather than with the expected content. +*/ +package atomicfile + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" +) + +// File is an io.ReadWriteCloser that can also be Canceled if a change needs to be abandoned. +type File interface { + io.ReadWriteCloser + // Cancel abandons a change to a file. This can be called if a write fails or another error occurs. + Cancel() error +} + +// ErrClosed is returned if Read or Write are called on a closed File. +var ErrClosed = errors.New("file is closed") + +// New returns a new atomic file. On Unix-like platforms, the writer (an io.ReadWriteCloser) is backed by a temporary +// file placed into the same directory as the destination file (using filepath.Dir to split the directory from the +// name). On a call to Close the temporary file is synced to disk and renamed to its final name, hiding any previous +// file by the same name. +// +// Note: Take care to call Close and handle any errors that are returned. Errors returned from Close may indicate that +// the file was not written with its final name. +func New(name string, mode os.FileMode) (File, error) { + return newFile(name, mode) +} + +type atomicFile struct { + name string + f *os.File + closed bool + closedMu sync.RWMutex +} + +func newFile(name string, mode os.FileMode) (File, error) { + dir := filepath.Dir(name) + f, err := os.CreateTemp(dir, "") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + if err := f.Chmod(mode); err != nil { + return nil, fmt.Errorf("failed to change temp file permissions: %w", err) + } + return &atomicFile{name: name, f: f}, nil +} + +func (a *atomicFile) Close() (err error) { + a.closedMu.Lock() + defer a.closedMu.Unlock() + + if a.closed { + return nil + } + a.closed = true + + defer func() { + if err != nil { + _ = os.Remove(a.f.Name()) // ignore errors + } + }() + // The order of operations here is: + // 1. sync + // 2. close + // 3. rename + // While the ordering of 2 and 3 is not important on Unix-like operating systems, Windows cannot rename an open + // file. By closing first, we allow the rename operation to succeed. + if err = a.f.Sync(); err != nil { + return fmt.Errorf("failed to sync temp file %q: %w", a.f.Name(), err) + } + if err = a.f.Close(); err != nil { + return fmt.Errorf("failed to close temp file %q: %w", a.f.Name(), err) + } + if err = os.Rename(a.f.Name(), a.name); err != nil { + return fmt.Errorf("failed to rename %q to %q: %w", a.f.Name(), a.name, err) + } + return nil +} + +func (a *atomicFile) Cancel() error { + a.closedMu.Lock() + defer a.closedMu.Unlock() + + if a.closed { + return nil + } + a.closed = true + _ = a.f.Close() // ignore error + return os.Remove(a.f.Name()) +} + +func (a *atomicFile) Read(p []byte) (n int, err error) { + a.closedMu.RLock() + defer a.closedMu.RUnlock() + if a.closed { + return 0, ErrClosed + } + return a.f.Read(p) +} + +func (a *atomicFile) Write(p []byte) (n int, err error) { + a.closedMu.RLock() + defer a.closedMu.RUnlock() + if a.closed { + return 0, ErrClosed + } + return a.f.Write(p) +} diff --git a/vendor/github.com/containerd/containerd/platforms/cpuinfo.go b/vendor/github.com/containerd/containerd/platforms/cpuinfo.go index 046e0356d19db..8c600fc96b1c7 100644 --- a/vendor/github.com/containerd/containerd/platforms/cpuinfo.go +++ b/vendor/github.com/containerd/containerd/platforms/cpuinfo.go @@ -17,14 +17,9 @@ package platforms import ( - "bufio" - "fmt" - "os" "runtime" - "strings" "sync" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" ) @@ -37,95 +32,12 @@ var cpuVariantOnce sync.Once func cpuVariant() string { cpuVariantOnce.Do(func() { if isArmArch(runtime.GOARCH) { - cpuVariantValue = getCPUVariant() + var err error + cpuVariantValue, err = getCPUVariant() + if err != nil { + log.L.Errorf("Error getCPUVariant for OS %s: %v", runtime.GOOS, err) + } } }) return cpuVariantValue } - -// For Linux, the kernel has already detected the ABI, ISA and Features. -// So we don't need to access the ARM registers to detect platform information -// by ourselves. We can just parse these information from /proc/cpuinfo -func getCPUInfo(pattern string) (info string, err error) { - if !isLinuxOS(runtime.GOOS) { - return "", fmt.Errorf("getCPUInfo for OS %s: %w", runtime.GOOS, errdefs.ErrNotImplemented) - } - - cpuinfo, err := os.Open("/proc/cpuinfo") - if err != nil { - return "", err - } - defer cpuinfo.Close() - - // Start to Parse the Cpuinfo line by line. For SMP SoC, we parse - // the first core is enough. - scanner := bufio.NewScanner(cpuinfo) - for scanner.Scan() { - newline := scanner.Text() - list := strings.Split(newline, ":") - - if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) { - return strings.TrimSpace(list[1]), nil - } - } - - // Check whether the scanner encountered errors - err = scanner.Err() - if err != nil { - return "", err - } - - return "", fmt.Errorf("getCPUInfo for pattern: %s: %w", pattern, errdefs.ErrNotFound) -} - -func getCPUVariant() string { - if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { - // Windows/Darwin only supports v7 for ARM32 and v8 for ARM64 and so we can use - // runtime.GOARCH to determine the variants - var variant string - switch runtime.GOARCH { - case "arm64": - variant = "v8" - case "arm": - variant = "v7" - default: - variant = "unknown" - } - - return variant - } - - variant, err := getCPUInfo("Cpu architecture") - if err != nil { - log.L.WithError(err).Error("failure getting variant") - return "" - } - - // handle edge case for Raspberry Pi ARMv6 devices (which due to a kernel quirk, report "CPU architecture: 7") - // https://www.raspberrypi.org/forums/viewtopic.php?t=12614 - if runtime.GOARCH == "arm" && variant == "7" { - model, err := getCPUInfo("model name") - if err == nil && strings.HasPrefix(strings.ToLower(model), "armv6-compatible") { - variant = "6" - } - } - - switch strings.ToLower(variant) { - case "8", "aarch64": - variant = "v8" - case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)": - variant = "v7" - case "6", "6tej": - variant = "v6" - case "5", "5t", "5te", "5tej": - variant = "v5" - case "4", "4t": - variant = "v4" - case "3": - variant = "v3" - default: - variant = "unknown" - } - - return variant -} diff --git a/vendor/github.com/containerd/containerd/platforms/cpuinfo_linux.go b/vendor/github.com/containerd/containerd/platforms/cpuinfo_linux.go new file mode 100644 index 0000000000000..722d86c3578cc --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/cpuinfo_linux.go @@ -0,0 +1,161 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import ( + "bufio" + "bytes" + "fmt" + "os" + "runtime" + "strings" + + "github.com/containerd/containerd/errdefs" + "golang.org/x/sys/unix" +) + +// getMachineArch retrieves the machine architecture through system call +func getMachineArch() (string, error) { + var uname unix.Utsname + err := unix.Uname(&uname) + if err != nil { + return "", err + } + + arch := string(uname.Machine[:bytes.IndexByte(uname.Machine[:], 0)]) + + return arch, nil +} + +// For Linux, the kernel has already detected the ABI, ISA and Features. +// So we don't need to access the ARM registers to detect platform information +// by ourselves. We can just parse these information from /proc/cpuinfo +func getCPUInfo(pattern string) (info string, err error) { + + cpuinfo, err := os.Open("/proc/cpuinfo") + if err != nil { + return "", err + } + defer cpuinfo.Close() + + // Start to Parse the Cpuinfo line by line. For SMP SoC, we parse + // the first core is enough. + scanner := bufio.NewScanner(cpuinfo) + for scanner.Scan() { + newline := scanner.Text() + list := strings.Split(newline, ":") + + if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) { + return strings.TrimSpace(list[1]), nil + } + } + + // Check whether the scanner encountered errors + err = scanner.Err() + if err != nil { + return "", err + } + + return "", fmt.Errorf("getCPUInfo for pattern %s: %w", pattern, errdefs.ErrNotFound) +} + +// getCPUVariantFromArch get CPU variant from arch through a system call +func getCPUVariantFromArch(arch string) (string, error) { + + var variant string + + arch = strings.ToLower(arch) + + if arch == "aarch64" { + variant = "8" + } else if arch[0:4] == "armv" && len(arch) >= 5 { + //Valid arch format is in form of armvXx + switch arch[3:5] { + case "v8": + variant = "8" + case "v7": + variant = "7" + case "v6": + variant = "6" + case "v5": + variant = "5" + case "v4": + variant = "4" + case "v3": + variant = "3" + default: + variant = "unknown" + } + } else { + return "", fmt.Errorf("getCPUVariantFromArch invalid arch: %s, %w", arch, errdefs.ErrInvalidArgument) + } + return variant, nil +} + +// getCPUVariant returns cpu variant for ARM +// We first try reading "Cpu architecture" field from /proc/cpuinfo +// If we can't find it, then fall back using a system call +// This is to cover running ARM in emulated environment on x86 host as this field in /proc/cpuinfo +// was not present. +func getCPUVariant() (string, error) { + + variant, err := getCPUInfo("Cpu architecture") + if err != nil { + if errdefs.IsNotFound(err) { + //Let's try getting CPU variant from machine architecture + arch, err := getMachineArch() + if err != nil { + return "", fmt.Errorf("failure getting machine architecture: %v", err) + } + + variant, err = getCPUVariantFromArch(arch) + if err != nil { + return "", fmt.Errorf("failure getting CPU variant from machine architecture: %v", err) + } + } else { + return "", fmt.Errorf("failure getting CPU variant: %v", err) + } + } + + // handle edge case for Raspberry Pi ARMv6 devices (which due to a kernel quirk, report "CPU architecture: 7") + // https://www.raspberrypi.org/forums/viewtopic.php?t=12614 + if runtime.GOARCH == "arm" && variant == "7" { + model, err := getCPUInfo("model name") + if err == nil && strings.HasPrefix(strings.ToLower(model), "armv6-compatible") { + variant = "6" + } + } + + switch strings.ToLower(variant) { + case "8", "aarch64": + variant = "v8" + case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)": + variant = "v7" + case "6", "6tej": + variant = "v6" + case "5", "5t", "5te", "5tej": + variant = "v5" + case "4", "4t": + variant = "v4" + case "3": + variant = "v3" + default: + variant = "unknown" + } + + return variant, nil +} diff --git a/vendor/github.com/containerd/containerd/platforms/cpuinfo_other.go b/vendor/github.com/containerd/containerd/platforms/cpuinfo_other.go new file mode 100644 index 0000000000000..51fb62ea7195b --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/cpuinfo_other.go @@ -0,0 +1,60 @@ +//go:build !linux +// +build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import ( + "fmt" + "runtime" + + "github.com/containerd/containerd/errdefs" +) + +func getCPUVariant() (string, error) { + + var variant string + + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + // Windows/Darwin only supports v7 for ARM32 and v8 for ARM64 and so we can use + // runtime.GOARCH to determine the variants + switch runtime.GOARCH { + case "arm64": + variant = "v8" + case "arm": + variant = "v7" + default: + variant = "unknown" + } + } else if runtime.GOOS == "freebsd" { + // FreeBSD supports ARMv6 and ARMv7 as well as ARMv4 and ARMv5 (though deprecated) + // detecting those variants is currently unimplemented + switch runtime.GOARCH { + case "arm64": + variant = "v8" + default: + variant = "unknown" + } + + } else { + return "", fmt.Errorf("getCPUVariant for OS %s: %v", runtime.GOOS, errdefs.ErrNotImplemented) + + } + + return variant, nil +} diff --git a/vendor/github.com/containerd/containerd/platforms/database.go b/vendor/github.com/containerd/containerd/platforms/database.go index dbe9957ca9dbd..2e26fd3b4faed 100644 --- a/vendor/github.com/containerd/containerd/platforms/database.go +++ b/vendor/github.com/containerd/containerd/platforms/database.go @@ -21,13 +21,6 @@ import ( "strings" ) -// isLinuxOS returns true if the operating system is Linux. -// -// The OS value should be normalized before calling this function. -func isLinuxOS(os string) bool { - return os == "linux" -} - // These function are generated from https://golang.org/src/go/build/syslist.go. // // We use switch statements because they are slightly faster than map lookups diff --git a/vendor/github.com/containerd/containerd/plugin/context.go b/vendor/github.com/containerd/containerd/plugin/context.go index dcb533c8a7454..cf91678988126 100644 --- a/vendor/github.com/containerd/containerd/plugin/context.go +++ b/vendor/github.com/containerd/containerd/plugin/context.go @@ -28,12 +28,13 @@ import ( // InitContext is used for plugin initialization type InitContext struct { - Context context.Context - Root string - State string - Config interface{} - Address string - TTRPCAddress string + Context context.Context + Root string + State string + Config interface{} + Address string + TTRPCAddress string + RegisterReadiness func() func() // deprecated: will be removed in 2.0, use plugin.EventType Events *exchange.Exchange diff --git a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go index 709fa028de278..13f500e4d2274 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go @@ -95,25 +95,30 @@ type ResolverOptions struct { Tracker StatusTracker // Authorizer is used to authorize registry requests - // Deprecated: use Hosts + // + // Deprecated: use Hosts. Authorizer Authorizer // Credentials provides username and secret given a host. // If username is empty but a secret is given, that secret // is interpreted as a long lived token. - // Deprecated: use Hosts + // + // Deprecated: use Hosts. Credentials func(string) (string, string, error) // Host provides the hostname given a namespace. - // Deprecated: use Hosts + // + // Deprecated: use Hosts. Host func(string) (string, error) // PlainHTTP specifies to use plain http and not https - // Deprecated: use Hosts + // + // Deprecated: use Hosts. PlainHTTP bool // Client is the http client to used when making registry requests - // Deprecated: use Hosts + // + // Deprecated: use Hosts. Client *http.Client } @@ -140,6 +145,9 @@ func NewResolver(options ResolverOptions) remotes.Resolver { if options.Headers == nil { options.Headers = make(http.Header) + } else { + // make a copy of the headers to avoid race due to concurrent map write + options.Headers = options.Headers.Clone() } if _, ok := options.Headers["User-Agent"]; !ok { options.Headers.Set("User-Agent", "containerd/"+version.Version) @@ -529,9 +537,10 @@ func (r *request) do(ctx context.Context) (*http.Response, error) { if err != nil { return nil, err } - req.Header = http.Header{} // headers need to be copied to avoid concurrent map access - for k, v := range r.header { - req.Header[k] = v + if r.header == nil { + req.Header = http.Header{} + } else { + req.Header = r.header.Clone() // headers need to be copied to avoid concurrent map access } if r.body != nil { body, err := r.body() diff --git a/vendor/github.com/containerd/containerd/runtime/v2/shim/util.go b/vendor/github.com/containerd/containerd/runtime/v2/shim/util.go index 1a0d41f23172e..94c0f5397855a 100644 --- a/vendor/github.com/containerd/containerd/runtime/v2/shim/util.go +++ b/vendor/github.com/containerd/containerd/runtime/v2/shim/util.go @@ -27,11 +27,13 @@ import ( "strings" "time" - "github.com/containerd/containerd/namespaces" "github.com/containerd/ttrpc" "github.com/gogo/protobuf/proto" "github.com/gogo/protobuf/types" exec "golang.org/x/sys/execabs" + + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/pkg/atomicfile" ) type CommandConfig struct { @@ -118,17 +120,16 @@ func WritePidFile(path string, pid int) error { if err != nil { return err } - tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path))) - f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666) + f, err := atomicfile.New(path, 0o666) if err != nil { return err } _, err = fmt.Fprintf(f, "%d", pid) - f.Close() if err != nil { + f.Cancel() return err } - return os.Rename(tempPath, path) + return f.Close() } // WriteAddress writes a address file atomically @@ -137,17 +138,16 @@ func WriteAddress(path, address string) error { if err != nil { return err } - tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path))) - f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666) + f, err := atomicfile.New(path, 0o666) if err != nil { return err } - _, err = f.WriteString(address) - f.Close() + _, err = f.Write([]byte(address)) if err != nil { + f.Cancel() return err } - return os.Rename(tempPath, path) + return f.Close() } // ErrNoAddress is returned when the address file has no content diff --git a/vendor/github.com/containerd/containerd/task.go b/vendor/github.com/containerd/containerd/task.go index 9be1394cf4643..ef8cd44942cfb 100644 --- a/vendor/github.com/containerd/containerd/task.go +++ b/vendor/github.com/containerd/containerd/task.go @@ -325,7 +325,16 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStat return nil, fmt.Errorf("task must be stopped before deletion: %s: %w", status.Status, errdefs.ErrFailedPrecondition) } if t.io != nil { - t.io.Close() + // io.Wait locks for restored tasks on Windows unless we call + // io.Close first (https://github.com/containerd/containerd/issues/5621) + // in other cases, preserve the contract and let IO finish before closing + if t.client.runtime == fmt.Sprintf("%s.%s", plugin.RuntimePlugin, "windows") { + t.io.Close() + } + // io.Cancel is used to cancel the io goroutine while it is in + // fifo-opening state. It does not stop the pipes since these + // should be closed on the shim's side, otherwise we might lose + // data from the container! t.io.Cancel() t.io.Wait() } diff --git a/vendor/github.com/containerd/containerd/version/version.go b/vendor/github.com/containerd/containerd/version/version.go index 2fee285ac1bc9..de124ef60aa5e 100644 --- a/vendor/github.com/containerd/containerd/version/version.go +++ b/vendor/github.com/containerd/containerd/version/version.go @@ -23,7 +23,7 @@ var ( Package = "github.com/containerd/containerd" // Version holds the complete version number. Filled in at linking time. - Version = "1.6.21+unknown" + Version = "1.6.22+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. diff --git a/vendor/modules.txt b/vendor/modules.txt index 4baa79bd2ac05..c3df1de562494 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -216,8 +216,8 @@ github.com/containerd/cgroups/v3/cgroup2/stats # github.com/containerd/console v1.0.3 ## explicit; go 1.13 github.com/containerd/console -# github.com/containerd/containerd v1.6.21 -## explicit; go 1.17 +# github.com/containerd/containerd v1.6.22 +## explicit; go 1.18 github.com/containerd/containerd github.com/containerd/containerd/api/events github.com/containerd/containerd/api/services/containers/v1 @@ -266,6 +266,7 @@ github.com/containerd/containerd/mount github.com/containerd/containerd/namespaces github.com/containerd/containerd/oci github.com/containerd/containerd/pkg/apparmor +github.com/containerd/containerd/pkg/atomicfile github.com/containerd/containerd/pkg/cap github.com/containerd/containerd/pkg/dialer github.com/containerd/containerd/pkg/kmutex From 72947f5022017ad11837ffa815023c2f2c70350c Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Tue, 11 Jul 2023 07:36:03 -0600 Subject: [PATCH 023/105] hack: use Git-free ROOTDIR convention Signed-off-by: Bjorn Neergaard (cherry picked from commit 5563b09ac242fb07e031e935c6f28f15baa41e62) Signed-off-by: Sebastiaan van Stijn --- hack/generate-authors.sh | 2 +- hack/validate/no-module | 2 +- hack/with-go-mod.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/generate-authors.sh b/hack/generate-authors.sh index 5133ed5b023c3..da30edb5fbded 100755 --- a/hack/generate-authors.sh +++ b/hack/generate-authors.sh @@ -3,7 +3,7 @@ set -e SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" +ROOTDIR="$(cd "${SCRIPTDIR}/.." && pwd)" set -x diff --git a/hack/validate/no-module b/hack/validate/no-module index 67a9c559add54..917cc3e756fbd 100755 --- a/hack/validate/no-module +++ b/hack/validate/no-module @@ -3,7 +3,7 @@ # Check that no one is trying to commit a go.mod. SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" +ROOTDIR="$(cd "${SCRIPTDIR}/../.." && pwd)" if test -e "${ROOTDIR}/go.mod"; then { diff --git a/hack/with-go-mod.sh b/hack/with-go-mod.sh index e4210f73c479a..868d204998b3c 100755 --- a/hack/with-go-mod.sh +++ b/hack/with-go-mod.sh @@ -9,7 +9,7 @@ set -e SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" +ROOTDIR="$(cd "${SCRIPTDIR}/.." && pwd)" if test -e "${ROOTDIR}/go.mod"; then { From 314b84b023acd11eca9b4a5c075cba1b77a6bd8a Mon Sep 17 00:00:00 2001 From: Kevin Alvarez Date: Tue, 11 Apr 2023 02:31:23 +0200 Subject: [PATCH 024/105] hack: enable Go modules when building dockerd and docker-proxy This is a workaround to have buildinfo with deps embedded in the binary. We need to create a go.mod file before building with -modfile=vendor.mod, otherwise it fails with: "-modfile cannot be used to set the module root directory." Signed-off-by: CrazyMax (cherry picked from commit 7665feeb528d02021b943aed3655eff5eca96598) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- hack/make/.binary | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4cbee5021570..0b32c3c901a17 100644 --- a/Dockerfile +++ b/Dockerfile @@ -599,7 +599,7 @@ RUN < "go.mod" + trap 'rm -f go.mod' EXIT + fi + echo "Building $([ "$DOCKER_STATIC" = "1" ] && echo "static" || echo "dynamic") $DEST/$BINARY_FULLNAME ($PLATFORM_NAME)..." if [ -n "$DOCKER_DEBUG" ]; then set -x fi - go build -o "$DEST/$BINARY_FULLNAME" "${BUILDFLAGS[@]}" -ldflags "$LDFLAGS $LDFLAGS_STATIC $DOCKER_LDFLAGS" ${GO_PACKAGE} + GO111MODULE=on go build -mod=vendor -modfile=vendor.mod -o "$DEST/$BINARY_FULLNAME" "${BUILDFLAGS[@]}" -ldflags "$LDFLAGS $LDFLAGS_STATIC $DOCKER_LDFLAGS" ${GO_PACKAGE} ) echo "Created binary: $DEST/$BINARY_FULLNAME" From e67f9dadc6cf5c7b4410e268d2a550cb18249f4a Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Tue, 11 Jul 2023 07:36:17 -0600 Subject: [PATCH 025/105] hack/make/.binary: use with-go-mod.sh Signed-off-by: Bjorn Neergaard (cherry picked from commit a972dbd682ec41e64a8ed95decf901ee040f003c) Signed-off-by: Sebastiaan van Stijn --- hack/make/.binary | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/hack/make/.binary b/hack/make/.binary index 90d16f11a250c..b73051f2a9531 100644 --- a/hack/make/.binary +++ b/hack/make/.binary @@ -74,19 +74,11 @@ source "${MAKEDIR}/.go-autogen" fi fi - # This is a workaround to have buildinfo with deps embedded in the binary. We - # need to create a go.mod file before building with -modfile=vendor.mod, - # otherwise it fails with: "-modfile cannot be used to set the module root directory." - if [ ! -f "go.mod" ]; then - printf '%s\n\n%s' 'module github.com/docker/docker' 'go 1.19' > "go.mod" - trap 'rm -f go.mod' EXIT - fi - echo "Building $([ "$DOCKER_STATIC" = "1" ] && echo "static" || echo "dynamic") $DEST/$BINARY_FULLNAME ($PLATFORM_NAME)..." if [ -n "$DOCKER_DEBUG" ]; then set -x fi - GO111MODULE=on go build -mod=vendor -modfile=vendor.mod -o "$DEST/$BINARY_FULLNAME" "${BUILDFLAGS[@]}" -ldflags "$LDFLAGS $LDFLAGS_STATIC $DOCKER_LDFLAGS" ${GO_PACKAGE} + ./hack/with-go-mod.sh go build -mod=vendor -modfile=vendor.mod -o "$DEST/$BINARY_FULLNAME" "${BUILDFLAGS[@]}" -ldflags "$LDFLAGS $LDFLAGS_STATIC $DOCKER_LDFLAGS" "$GO_PACKAGE" ) echo "Created binary: $DEST/$BINARY_FULLNAME" From bf2b8a05a0b7bd07eb35ab73d4e6af50651a8625 Mon Sep 17 00:00:00 2001 From: Luboslav Pivarc Date: Wed, 10 May 2023 10:09:21 +0200 Subject: [PATCH 026/105] Do not drop effective&permitted set Currently moby drops ep sets before the entrypoint is executed. This does mean that with combination of no-new-privileges the file capabilities stops working with non-root containers. This is undesired as the usability of such containers is harmed comparing to running root containers. This commit therefore sets the effective/permitted set in order to allow use of file capabilities or libcap(3)/prctl(2) respectively with combination of no-new-privileges and without respectively. For no-new-privileges the container will be able to obtain capabilities that are requested. Signed-off-by: Luboslav Pivarc Signed-off-by: Bjorn Neergaard (cherry picked from commit 3aef732e61ec8ae0ea0bd8ad31116194e0fc21a6) Signed-off-by: Sebastiaan van Stijn --- oci/oci.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/oci/oci.go b/oci/oci.go index 864ccf5b60c41..45ed7979ee827 100644 --- a/oci/oci.go +++ b/oci/oci.go @@ -23,19 +23,10 @@ func SetCapabilities(s *specs.Spec, caplist []string) error { if s.Process == nil { s.Process = &specs.Process{} } - // setUser has already been executed here - if s.Process.User.UID == 0 { - s.Process.Capabilities = &specs.LinuxCapabilities{ - Effective: caplist, - Bounding: caplist, - Permitted: caplist, - } - } else { - // Do not set Effective and Permitted capabilities for non-root users, - // to match what execve does. - s.Process.Capabilities = &specs.LinuxCapabilities{ - Bounding: caplist, - } + s.Process.Capabilities = &specs.LinuxCapabilities{ + Effective: caplist, + Bounding: caplist, + Permitted: caplist, } return nil } From cc39fb9f6b6863998f061f303f0a10377336cc6b Mon Sep 17 00:00:00 2001 From: Luboslav Pivarc Date: Mon, 17 Jul 2023 11:48:02 +0200 Subject: [PATCH 027/105] Integration test for capabilities Verify non-root containers are able to use file capabilities. Signed-off-by: Luboslav Pivarc Co-authored-by: Cory Snider Signed-off-by: Cory Snider (cherry picked from commit 42fa7a1951f192a0038b904f529e86beeb056093) Signed-off-by: Sebastiaan van Stijn --- .../capabilities/capabilities_linux_test.go | 108 ++++++++++++++++++ integration/capabilities/main_linux_test.go | 33 ++++++ integration/internal/container/ops.go | 18 +++ 3 files changed, 159 insertions(+) create mode 100644 integration/capabilities/capabilities_linux_test.go create mode 100644 integration/capabilities/main_linux_test.go diff --git a/integration/capabilities/capabilities_linux_test.go b/integration/capabilities/capabilities_linux_test.go new file mode 100644 index 0000000000000..272f3dcdb7c5c --- /dev/null +++ b/integration/capabilities/capabilities_linux_test.go @@ -0,0 +1,108 @@ +package capabilities + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/docker/testutil/fakecontext" + + "gotest.tools/v3/assert" + "gotest.tools/v3/poll" +) + +func TestNoNewPrivileges(t *testing.T) { + defer setupTest(t)() + + withFileCapability := ` + FROM debian:bullseye-slim + RUN apt-get update && apt-get install -y libcap2-bin --no-install-recommends + RUN setcap CAP_DAC_OVERRIDE=+eip /bin/cat + RUN echo "hello" > /txt && chown 0:0 /txt && chmod 700 /txt + RUN useradd -u 1500 test + ` + imageTag := "captest" + + source := fakecontext.New(t, "", fakecontext.WithDockerfile(withFileCapability)) + defer source.Close() + + client := testEnv.APIClient() + + // Build image + ctx := context.TODO() + resp, err := client.ImageBuild(ctx, + source.AsTarReader(t), + types.ImageBuildOptions{ + Tags: []string{imageTag}, + }) + assert.NilError(t, err) + _, err = io.Copy(io.Discard, resp.Body) + assert.NilError(t, err) + resp.Body.Close() + + testCases := []struct { + doc string + opts []func(*container.TestContainerConfig) + stdOut, stdErr string + }{ + { + doc: "CapabilityRequested=true", + opts: []func(*container.TestContainerConfig){ + container.WithUser("test"), + container.WithCapability("CAP_DAC_OVERRIDE"), + }, + stdOut: "hello", + }, + { + doc: "CapabilityRequested=false", + opts: []func(*container.TestContainerConfig){ + container.WithUser("test"), + container.WithDropCapability("CAP_DAC_OVERRIDE"), + }, + stdErr: "exec /bin/cat: operation not permitted", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.doc, func(t *testing.T) { + // Run the container with the image + opts := append(tc.opts, + container.WithImage(imageTag), + container.WithCmd("/bin/cat", "/txt"), + container.WithSecurityOpt("no-new-privileges=true"), + ) + cid := container.Run(ctx, t, client, opts...) + poll.WaitOn(t, container.IsInState(ctx, client, cid, "exited"), poll.WithDelay(100*time.Millisecond)) + + // Assert on outputs + logReader, err := client.ContainerLogs(ctx, cid, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + assert.NilError(t, err) + defer logReader.Close() + + var actualStdout, actualStderr bytes.Buffer + _, err = stdcopy.StdCopy(&actualStdout, &actualStderr, logReader) + assert.NilError(t, err) + + stdOut := strings.TrimSpace(actualStdout.String()) + stdErr := strings.TrimSpace(actualStderr.String()) + if stdOut != tc.stdOut { + t.Fatalf("test produced invalid output: %q, expected %q. Stderr:%q", stdOut, tc.stdOut, stdErr) + } + if stdErr != tc.stdErr { + t.Fatalf("test produced invalid error: %q, expected %q. Stdout:%q", stdErr, tc.stdErr, stdOut) + + } + }) + } + +} diff --git a/integration/capabilities/main_linux_test.go b/integration/capabilities/main_linux_test.go new file mode 100644 index 0000000000000..0f074dd15603e --- /dev/null +++ b/integration/capabilities/main_linux_test.go @@ -0,0 +1,33 @@ +package capabilities + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/testutil/environment" +) + +var testEnv *environment.Execution + +func TestMain(m *testing.M) { + var err error + testEnv, err = environment.New() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + err = environment.EnsureFrozenImagesLinux(testEnv) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + testEnv.Print() + os.Exit(m.Run()) +} + +func setupTest(t *testing.T) func() { + environment.ProtectAll(t, testEnv) + return func() { testEnv.Clean(t) } +} diff --git a/integration/internal/container/ops.go b/integration/internal/container/ops.go index 33d977700623c..838f515e9f24e 100644 --- a/integration/internal/container/ops.go +++ b/integration/internal/container/ops.go @@ -237,3 +237,21 @@ func WithRuntime(name string) func(*TestContainerConfig) { c.HostConfig.Runtime = name } } + +func WithCapability(capabilities ...string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.CapAdd = append(c.HostConfig.CapAdd, capabilities...) + } +} + +func WithDropCapability(capabilities ...string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.CapDrop = append(c.HostConfig.CapDrop, capabilities...) + } +} + +func WithSecurityOpt(opt string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.SecurityOpt = append(c.HostConfig.SecurityOpt, opt) + } +} From 15bd07b4fda2f02eca528e6e437a34bcb4bbef74 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 11 Aug 2023 21:29:53 +0900 Subject: [PATCH 028/105] update runc binary to v1.1.9 Signed-off-by: Akihiro Suda (cherry picked from commit b039bbc678892661e2e42fd69d7aa417887b8ecf) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- hack/dockerfile/install/runc.installer | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4cbee5021570..ef96a665af8ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -280,7 +280,7 @@ RUN git init . && git remote add origin "https://github.com/opencontainers/runc. # that is used. If you need to update runc, open a pull request in the containerd # project first, and update both after that is merged. When updating RUNC_VERSION, # consider updating runc in vendor.mod accordingly. -ARG RUNC_VERSION=v1.1.8 +ARG RUNC_VERSION=v1.1.9 RUN git fetch -q --depth 1 origin "${RUNC_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD FROM base AS runc-build diff --git a/hack/dockerfile/install/runc.installer b/hack/dockerfile/install/runc.installer index 289a0ca9ff478..3ac1ddfbf5f9f 100755 --- a/hack/dockerfile/install/runc.installer +++ b/hack/dockerfile/install/runc.installer @@ -9,7 +9,7 @@ set -e # the containerd project first, and update both after that is merged. # # When updating RUNC_VERSION, consider updating runc in vendor.mod accordingly -: "${RUNC_VERSION:=v1.1.8}" +: "${RUNC_VERSION:=v1.1.9}" install_runc() { RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp"}" From 5d2c383d72f45ec438d39ce52f1eac6490ca660c Mon Sep 17 00:00:00 2001 From: Sam Thibault Date: Mon, 14 Aug 2023 14:03:52 +0200 Subject: [PATCH 029/105] remove s390x and ppc64ls pipelines Signed-off-by: Sam Thibault (cherry picked from commit 59aa3dce8a3abc9d6eb29df7c76b4484e8758234) Signed-off-by: Sebastiaan van Stijn --- Jenkinsfile | 402 ---------------------------------------------------- 1 file changed, 402 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6707839168792..d93a24a89f08e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,8 +9,6 @@ pipeline { } parameters { booleanParam(name: 'arm64', defaultValue: true, description: 'ARM (arm64) Build/Test') - booleanParam(name: 's390x', defaultValue: false, description: 'IBM Z (s390x) Build/Test') - booleanParam(name: 'ppc64le', defaultValue: false, description: 'PowerPC (ppc64le) Build/Test') booleanParam(name: 'dco', defaultValue: true, description: 'Run the DCO check') } environment { @@ -51,406 +49,6 @@ pipeline { } stage('Build') { parallel { - stage('s390x') { - when { - beforeAgent true - // Skip this stage on PRs unless the checkbox is selected - anyOf { - not { changeRequest() } - expression { params.s390x } - } - } - agent { label 's390x-ubuntu-2004' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker build --force-rm -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Unit tests") { - steps { - sh ''' - sudo modprobe ip6table_filter - ''' - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/test/unit - ''' - } - post { - always { - junit testResults: 'bundles/junit-report*.xml', allowEmptyResults: true - } - } - } - stage("Integration tests") { - environment { TEST_SKIP_INTEGRATION_CLI = '1' } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TESTDEBUG \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ - -e TEST_SKIP_INTEGRATION_CLI \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=s390x-integration - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - stage('s390x integration-cli') { - when { - beforeAgent true - // Skip this stage on PRs unless the checkbox is selected - anyOf { - not { changeRequest() } - expression { params.s390x } - } - } - agent { label 's390x-ubuntu-2004' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker build --force-rm -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Integration-cli tests") { - environment { TEST_SKIP_INTEGRATION = '1' } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ - -e TEST_SKIP_INTEGRATION \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=s390x-integration-cli - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - stage('ppc64le') { - when { - beforeAgent true - // Skip this stage on PRs unless the checkbox is selected - anyOf { - not { changeRequest() } - expression { params.ppc64le } - } - } - agent { label 'ppc64le-ubuntu-1604' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker buildx build --load --force-rm -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Unit tests") { - steps { - sh ''' - sudo modprobe ip6table_filter - ''' - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/test/unit - ''' - } - post { - always { - junit testResults: 'bundles/junit-report*.xml', allowEmptyResults: true - } - } - } - stage("Integration tests") { - environment { TEST_SKIP_INTEGRATION_CLI = '1' } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_EXPERIMENTAL \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TESTDEBUG \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ - -e TEST_SKIP_INTEGRATION_CLI \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=ppc64le-integration - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } - stage('ppc64le integration-cli') { - when { - beforeAgent true - // Skip this stage on PRs unless the checkbox is selected - anyOf { - not { changeRequest() } - expression { params.ppc64le } - } - } - agent { label 'ppc64le-ubuntu-1604' } - - stages { - stage("Print info") { - steps { - sh 'docker version' - sh 'docker info' - sh ''' - echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}" - curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \ - && bash ${WORKSPACE}/check-config.sh || true - ''' - } - } - stage("Build dev image") { - steps { - sh ''' - docker buildx build --load --force-rm -t docker:${GIT_COMMIT} . - ''' - } - } - stage("Integration-cli tests") { - environment { TEST_SKIP_INTEGRATION = '1' } - steps { - sh ''' - docker run --rm -t --privileged \ - -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \ - --name docker-pr$BUILD_NUMBER \ - -e DOCKER_GITCOMMIT=${GIT_COMMIT} \ - -e DOCKER_GRAPHDRIVER \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ - -e TEST_SKIP_INTEGRATION \ - -e TIMEOUT \ - -e VALIDATE_REPO=${GIT_URL} \ - -e VALIDATE_BRANCH=${CHANGE_TARGET} \ - docker:${GIT_COMMIT} \ - hack/make.sh \ - dynbinary \ - test-integration - ''' - } - post { - always { - junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true - } - } - } - } - - post { - always { - sh ''' - echo "Ensuring container killed." - docker rm -vf docker-pr$BUILD_NUMBER || true - ''' - - sh ''' - echo "Chowning /workspace to jenkins user" - docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace - ''' - - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') { - sh ''' - bundleName=ppc64le-integration-cli - echo "Creating ${bundleName}-bundles.tar.gz" - # exclude overlay2 directories - find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz - ''' - - archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true - } - } - cleanup { - sh 'make clean' - deleteDir() - } - } - } stage('arm64') { when { beforeAgent true From 600aa7b7a52a450fc4e1daeceea1636f43b1f6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 16 Aug 2023 11:16:50 +0200 Subject: [PATCH 030/105] c8d/inspect: Ignore manifest with missing config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a failure to inspect image if any of its present manifest references an image config which isn't present locally. Signed-off-by: Paweł Gronowski (cherry picked from commit a64adda4e77890f8b126cf7a723f27735a07e5d2) Signed-off-by: Paweł Gronowski --- daemon/containerd/image.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index 7725476d1fbd1..eaa94b8155da5 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -48,11 +48,25 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima err = i.walkImageManifests(ctx, desc, func(img *ImageManifest) error { conf, err := img.Config(ctx) if err != nil { - return err + if cerrdefs.IsNotFound(err) { + logrus.WithFields(logrus.Fields{ + "manifestDescriptor": img.Target(), + }).Debug("manifest was present, but accessing its config failed, ignoring") + return nil + } + return errdefs.System(fmt.Errorf("failed to get config descriptor: %w", err)) } + var ociimage ocispec.Image if err := readConfig(ctx, cs, conf, &ociimage); err != nil { - return err + if cerrdefs.IsNotFound(err) { + logrus.WithFields(logrus.Fields{ + "manifestDescriptor": img.Target(), + "configDescriptor": conf, + }).Debug("manifest present, but its config is missing, ignoring") + return nil + } + return errdefs.System(fmt.Errorf("failed to read config of the manifest %v: %w", img.Target().Digest, err)) } presentImages = append(presentImages, ociimage) return nil @@ -61,7 +75,8 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima return nil, err } if len(presentImages) == 0 { - return nil, errdefs.NotFound(errors.New("failed to find image manifest")) + ref, _ := reference.ParseAnyReference(refOrID) + return nil, images.ErrImageDoesNotExist{Ref: ref} } sort.SliceStable(presentImages, func(i, j int) bool { From 3ce0dc7e35dcb8466b73e22c7f28e69acd2d8afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 31 Jul 2023 18:32:01 +0200 Subject: [PATCH 031/105] bakefile: Remove default value of DOCKER_GITCOMMIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "HEAD" will still be used as a version if no DOCKER_COMMIT is provided (for example when not running via `make`), but it won't prevent it being set to the GITHUB_SHA variable when it's present. This should fix `Git commit` reported by `docker version` for the binaries generated by `moby-bin`. Signed-off-by: Paweł Gronowski (cherry picked from commit d7a9f15775e023f5675a48381c3d7d78c25dd16d) Signed-off-by: Paweł Gronowski --- docker-bake.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 3d9675f184431..a1ac477a91b08 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -14,7 +14,7 @@ variable "DOCKER_BUILDTAGS" { default = "" } variable "DOCKER_GITCOMMIT" { - default = "HEAD" + default = null } # Docker version such as 23.0.0-dev. Automatically generated through Git ref. @@ -81,7 +81,7 @@ target "_common" { DOCKER_STATIC = DOCKER_STATIC DOCKER_LDFLAGS = DOCKER_LDFLAGS DOCKER_BUILDTAGS = DOCKER_BUILDTAGS - DOCKER_GITCOMMIT = DOCKER_GITCOMMIT != "" ? DOCKER_GITCOMMIT : GITHUB_SHA + DOCKER_GITCOMMIT = DOCKER_GITCOMMIT != null ? DOCKER_GITCOMMIT : GITHUB_SHA VERSION = VERSION != "" ? VERSION : GITHUB_REF PLATFORM = PLATFORM PRODUCT = PRODUCT From 448ae33f875e43ddcd30c8cadfbafb2cbd942637 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Thu, 17 Aug 2023 11:34:38 -0600 Subject: [PATCH 032/105] ci(bin-image): populate DOCKER_GITCOMMIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bjorn Neergaard (cherry picked from commit 9aed6308d42eecabc78900c3c68488b75eff0337) Signed-off-by: Paweł Gronowski --- .github/workflows/bin-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index 182f02c04f999..ffdd25fb6bb7d 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -118,6 +118,8 @@ jobs: ./docker-bake.hcl /tmp/bake-meta.json targets: bin-image + env: + DOCKER_GITCOMMIT: ${{ github.sha }} set: | *.platform=${{ matrix.platform }} *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} From 0c131f58bae16688926e0af72790dfec22ec3b4b Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Thu, 17 Aug 2023 13:17:33 -0600 Subject: [PATCH 033/105] ci(bin-image): populate DOCKER_GITCOMMIT, take 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bjorn Neergaard (cherry picked from commit 73ffb48bfbbc1fb927c689914cfff5992129da0f) Signed-off-by: Paweł Gronowski --- .github/workflows/bin-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index ffdd25fb6bb7d..aa6d41130cdcb 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -113,13 +113,13 @@ jobs: name: Build id: bake uses: docker/bake-action@v3 + env: + DOCKER_GITCOMMIT: ${{ github.sha }} with: files: | ./docker-bake.hcl /tmp/bake-meta.json targets: bin-image - env: - DOCKER_GITCOMMIT: ${{ github.sha }} set: | *.platform=${{ matrix.platform }} *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} From 3897724f4a7c036b3b1b7ecb2feb66056696f6e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Fri, 18 Aug 2023 11:38:50 +0200 Subject: [PATCH 034/105] volume/local: Fix debug log typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Active count is incremented, but message claimed the opposite. Signed-off-by: Paweł Gronowski (cherry picked from commit 7f965d55c719b8777221c4dbf9a1eb93cfada8ab) Signed-off-by: Paweł Gronowski --- volume/local/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volume/local/local.go b/volume/local/local.go index f156ea2339bc3..1417941cca3a7 100644 --- a/volume/local/local.go +++ b/volume/local/local.go @@ -310,7 +310,7 @@ func (v *localVolume) Mount(id string) (string, error) { v.active.mounted = true } v.active.count++ - logger.WithField("active mounts", v.active).Debug("Decremented active mount count") + logger.WithField("active mounts", v.active).Debug("Incremented active mount count") } if err := v.postMount(); err != nil { return "", err From 54953f2f5a5157a9ae92d25831a2b77c20cfab82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 10 Aug 2023 19:13:38 +0200 Subject: [PATCH 035/105] integration: Add test for not breaking overlayfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check that operations that could potentially perform overlayfs mounts that could cause undefined behaviors. Signed-off-by: Paweł Gronowski (cherry picked from commit 303e2b124e6697a232d0c1a5207cddd79e561fe1) Signed-off-by: Paweł Gronowski --- integration/container/overlayfs_linux_test.go | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 integration/container/overlayfs_linux_test.go diff --git a/integration/container/overlayfs_linux_test.go b/integration/container/overlayfs_linux_test.go new file mode 100644 index 0000000000000..4736e6afc22f1 --- /dev/null +++ b/integration/container/overlayfs_linux_test.go @@ -0,0 +1,111 @@ +package container + +import ( + "context" + "io" + "strings" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/dmesg" + "gotest.tools/v3/assert" + "gotest.tools/v3/skip" +) + +func TestNoOverlayfsWarningsAboutUndefinedBehaviors(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType != "linux", "overlayfs is only available on linux") + skip.If(t, testEnv.IsRemoteDaemon(), "local daemon is needed for kernel log access") + skip.If(t, testEnv.IsRootless(), "root is needed for reading kernel log") + + defer setupTest(t)() + client := testEnv.APIClient() + ctx := context.Background() + + cID := container.Run(ctx, t, client, container.WithCmd("sh", "-c", `while true; do echo $RANDOM >>/file; sleep 0.1; done`)) + + testCases := []struct { + name string + operation func(t *testing.T) error + }{ + {name: "diff", operation: func(*testing.T) error { + _, err := client.ContainerDiff(ctx, cID) + return err + }}, + {name: "export", operation: func(*testing.T) error { + rc, err := client.ContainerExport(ctx, cID) + if err == nil { + defer rc.Close() + _, err = io.Copy(io.Discard, rc) + } + return err + }}, + {name: "cp to container", operation: func(t *testing.T) error { + archive, err := archive.Generate("new-file", "hello-world") + assert.NilError(t, err, "failed to create a temporary archive") + return client.CopyToContainer(ctx, cID, "/", archive, types.CopyToContainerOptions{}) + }}, + {name: "cp from container", operation: func(*testing.T) error { + rc, _, err := client.CopyFromContainer(ctx, cID, "/file") + if err == nil { + defer rc.Close() + _, err = io.Copy(io.Discard, rc) + } + + return err + }}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + prev := dmesgLines(256) + + err := tc.operation(t) + assert.NilError(t, err) + + after := dmesgLines(2048) + + diff := diffDmesg(prev, after) + for _, line := range diff { + overlayfs := strings.Contains(line, "overlayfs: ") + lowerDirInUse := strings.Contains(line, "lowerdir is in-use as ") + upperDirInUse := strings.Contains(line, "upperdir is in-use as ") + workDirInuse := strings.Contains(line, "workdir is in-use as ") + undefinedBehavior := strings.Contains(line, "will result in undefined behavior") + + if overlayfs && (lowerDirInUse || upperDirInUse || workDirInuse) && undefinedBehavior { + t.Errorf("%s caused overlayfs kernel warning: %s", tc.name, line) + } + } + }) + } +} + +func dmesgLines(bytes int) []string { + data := dmesg.Dmesg(bytes) + return strings.Split(strings.TrimSpace(string(data)), "\n") +} + +func diffDmesg(prev, next []string) []string { + // All lines have a timestamp, so just take the last one from the previous + // log and find it in the new log. + lastPrev := prev[len(prev)-1] + + for idx := len(next) - 1; idx >= 0; idx-- { + line := next[idx] + + if line == lastPrev { + nextIdx := idx + 1 + if nextIdx < len(next) { + return next[nextIdx:] + } else { + // Found at the last position, log is the same. + return nil + } + } + } + + return next +} From b76a0c7d009c9859eb01fce680685cb55b4a36bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 10 Aug 2023 15:33:06 +0200 Subject: [PATCH 036/105] c8d/export: Use ref counted mounter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To prevent mounting the container rootfs in a rw mode if it's already mounted. This can't use `mount.WithReadonlyTempMount` because the archive code does a chroot with a pivot_root, which creates a new directory in the rootfs. Signed-off-by: Paweł Gronowski (cherry picked from commit 051d51b22212bb570998e8cf3593036177c4a647) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_exporter.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/daemon/containerd/image_exporter.go b/daemon/containerd/image_exporter.go index 21bbcfc48ef4c..9dfb3e6a0ec73 100644 --- a/daemon/containerd/image_exporter.go +++ b/daemon/containerd/image_exporter.go @@ -11,7 +11,6 @@ import ( containerdimages "github.com/containerd/containerd/images" "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/leases" - "github.com/containerd/containerd/mount" cplatforms "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" "github.com/docker/docker/container" @@ -30,7 +29,13 @@ func (i *ImageService) PerformWithBaseFS(ctx context.Context, c *container.Conta if err != nil { return err } - return mount.WithTempMount(ctx, mounts, fn) + path, err := i.refCountMounter.Mount(mounts, c.ID) + if err != nil { + return err + } + defer i.refCountMounter.Unmount(path) + + return fn(path) } // ExportImage exports a list of images to the given output stream. The From 74bf46aea6e17fe088e5efa6647c4fb558b22398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 10 Aug 2023 19:05:03 +0200 Subject: [PATCH 037/105] c8d/diff: Reuse mount, mount parent as read-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The container rw layer may already be mounted, so it's not safe to use it in another overlay mount. Use the ref counted mounter (which will reuse the existing mount if it exists) to avoid that. Also, mount the parent mounts (layers of the base image) in a read-only mode. Signed-off-by: Paweł Gronowski (cherry picked from commit 6da42ca8308fccf3ae5be75d599eeda61f7e41ed) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_changes.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/daemon/containerd/image_changes.go b/daemon/containerd/image_changes.go index 94c56cf6161c6..dd44aebe80717 100644 --- a/daemon/containerd/image_changes.go +++ b/daemon/containerd/image_changes.go @@ -58,15 +58,10 @@ func (i *ImageService) Changes(ctx context.Context, container *container.Contain } }() - mounts, err := snapshotter.Mounts(ctx, container.ID) - if err != nil { - return nil, err - } - var changes []archive.Change - err = mount.WithReadonlyTempMount(ctx, mounts, func(fs string) error { - return mount.WithTempMount(ctx, parent, func(root string) error { - changes, err = archive.ChangesDirs(fs, root) + err = i.PerformWithBaseFS(ctx, container, func(containerRootfs string) error { + return mount.WithReadonlyTempMount(ctx, parent, func(parentRootfs string) error { + changes, err = archive.ChangesDirs(containerRootfs, parentRootfs) return err }) }) From fb6784bdf0bdbc3ff59052632f3121e7c556d697 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 21 Aug 2023 18:51:41 +0200 Subject: [PATCH 038/105] gha: set 10-minute timeout on "report" actions I had a CI run fail to "Upload reports": Exponential backoff for retry #1. Waiting for 4565 milliseconds before continuing the upload at offset 0 Finished backoff for retry #1, continuing with upload Total file count: 211 ---- Processed file #160 (75.8%) ... Total file count: 211 ---- Processed file #164 (77.7%) Total file count: 211 ---- Processed file #164 (77.7%) Total file count: 211 ---- Processed file #164 (77.7%) A 503 status code has been received, will attempt to retry the upload ##### Begin Diagnostic HTTP information ##### Status Code: 503 Status Message: Service Unavailable Header Information: { "content-length": "592", "content-type": "application/json; charset=utf-8", "date": "Mon, 21 Aug 2023 14:08:10 GMT", "server": "Kestrel", "cache-control": "no-store,no-cache", "pragma": "no-cache", "strict-transport-security": "max-age=2592000", "x-tfs-processid": "b2fc902c-011a-48be-858d-c62e9c397cb6", "activityid": "49a48b53-0411-4ff3-86a7-4528e3f71ba2", "x-tfs-session": "49a48b53-0411-4ff3-86a7-4528e3f71ba2", "x-vss-e2eid": "49a48b53-0411-4ff3-86a7-4528e3f71ba2", "x-vss-senderdeploymentid": "63be6134-28d1-8c82-e969-91f4e88fcdec", "x-frame-options": "SAMEORIGIN" } ###### End Diagnostic HTTP information ###### Retry limit has been reached for chunk at offset 0 to https://pipelinesghubeus5.actions.githubusercontent.com/Y2huPMnV2RyiTvKoReSyXTCrcRyxUdSDRZYoZr0ONBvpl5e9Nu/_apis/resources/Containers/8331549?itemPath=integration-reports%2Fubuntu-22.04-systemd%2Fbundles%2Ftest-integration%2FTestInfoRegistryMirrors%2Fd20ac12e48cea%2Fdocker.log Warning: Aborting upload for /tmp/reports/ubuntu-22.04-systemd/bundles/test-integration/TestInfoRegistryMirrors/d20ac12e48cea/docker.log due to failure Error: aborting artifact upload Total file count: 211 ---- Processed file #165 (78.1%) A 503 status code has been received, will attempt to retry the upload Exponential backoff for retry #1. Waiting for 5799 milliseconds before continuing the upload at offset 0 As a result, the "Download reports" continued retrying: ... Total file count: 1004 ---- Processed file #436 (43.4%) Total file count: 1004 ---- Processed file #436 (43.4%) Total file count: 1004 ---- Processed file #436 (43.4%) An error occurred while attempting to download a file Error: Request timeout: /Y2huPMnV2RyiTvKoReSyXTCrcRyxUdSDRZYoZr0ONBvpl5e9Nu/_apis/resources/Containers/8331549?itemPath=integration-reports%2Fubuntu-20.04%2Fbundles%2Ftest-integration%2FTestCreateWithDuplicateNetworkNames%2Fd47798cc212d1%2Fdocker.log at ClientRequest. (/home/runner/work/_actions/actions/download-artifact/v3/dist/index.js:3681:26) at Object.onceWrapper (node:events:627:28) at ClientRequest.emit (node:events:513:28) at TLSSocket.emitRequestTimeout (node:_http_client:839:9) at Object.onceWrapper (node:events:627:28) at TLSSocket.emit (node:events:525:35) at TLSSocket.Socket._onTimeout (node:net:550:8) at listOnTimeout (node:internal/timers:559:17) at processTimers (node:internal/timers:502:7) Exponential backoff for retry #1. Waiting for 5305 milliseconds before continuing the download Total file count: 1004 ---- Processed file #436 (43.4%) And, it looks like GitHub doesn't allow cancelling the job, possibly because it is defined with `if: always()`? Signed-off-by: Sebastiaan van Stijn (cherry picked from commit d6f340e784ee2b4e208be9834f75f042a0fb6691) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e7467f418c75..73d3092b7f52b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,6 +166,7 @@ jobs: unit-report: runs-on: ubuntu-20.04 + timeout-minutes: 10 if: always() needs: - unit @@ -354,6 +355,7 @@ jobs: integration-report: runs-on: ubuntu-20.04 + timeout-minutes: 10 if: always() needs: - integration @@ -482,6 +484,7 @@ jobs: integration-cli-report: runs-on: ubuntu-20.04 + timeout-minutes: 10 if: always() needs: - integration-cli From bb22b8a41876ab3df411501994199c0cd21f7ed5 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 19 Aug 2023 07:19:10 +0200 Subject: [PATCH 039/105] ci(bin-image): check repo origin Signed-off-by: CrazyMax (cherry picked from commit 219d4d9db9941e965c21eca045b7472d43b5599c) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/bin-image.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index aa6d41130cdcb..19286d226a90c 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -104,7 +104,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.repository == 'moby/moby' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_MOBYBIN_USERNAME }} @@ -122,18 +122,18 @@ jobs: targets: bin-image set: | *.platform=${{ matrix.platform }} - *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} + *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' && github.repository == 'moby/moby' }} *.tags= - name: Export digest - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.repository == 'moby/moby' run: | mkdir -p /tmp/digests digest="${{ fromJSON(steps.bake.outputs.metadata)['bin-image']['containerimage.digest'] }}" touch "/tmp/digests/${digest#sha256:}" - name: Upload digest - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.repository == 'moby/moby' uses: actions/upload-artifact@v3 with: name: digests @@ -143,7 +143,7 @@ jobs: merge: runs-on: ubuntu-20.04 - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && github.repository == 'moby/moby' needs: - build steps: From e23979958366b0157257f7baab23fce19b5bc958 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 31 Jul 2023 16:14:43 +0200 Subject: [PATCH 040/105] distribution: update warning for deprecated image formats - Use the same warning for both "v1 in manifest-index" and bare "v1" images. - Update URL to use a "/go/" redirect, which allows the docs team to more easily redirect the URL to relevant docs (if things move). Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 982bc0e228e89c3117cc326301ff5b4b9bef3a8a) Signed-off-by: Sebastiaan van Stijn --- distribution/pull_v2.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index 06e10a9b8d54b..2277d4e58fdc4 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -441,7 +441,7 @@ func (p *puller) pullTag(ctx context.Context, ref reference.Named, platform *oci // give registries time to upgrade to schema2 and only warn if we know a registry has been upgraded long time ago // TODO: condition to be removed if reference.Domain(ref) == "docker.io" { - msg := fmt.Sprintf("Image %s uses outdated schema1 manifest format. Please upgrade to a schema2 image for better future compatibility. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/", ref) + msg := fmt.Sprintf("[DEPRECATION NOTICE] Docker Image Format v1, and Docker Image manifest version 2, schema 1 support will be removed in an upcoming release. Suggest the author of %s to upgrade the image to the OCI Format, or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/", ref) logrus.Warn(msg) progress.Message(p.config.ProgressOutput, "", msg) } @@ -873,7 +873,7 @@ func (p *puller) pullManifestList(ctx context.Context, ref reference.Named, mfst switch v := manifest.(type) { case *schema1.SignedManifest: - msg := fmt.Sprintf("[DEPRECATION NOTICE] v2 schema1 manifests in manifest lists are not supported and will break in a future release. Suggest author of %s to upgrade to v2 schema2. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/", ref) + msg := fmt.Sprintf("[DEPRECATION NOTICE] Docker Image Format v1, and Docker Image manifest version 2, schema 1 support will be removed in an upcoming release. Suggest the author of %s to upgrade the image to the OCI Format, or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/", ref) logrus.Warn(msg) progress.Message(p.config.ProgressOutput, "", msg) From a99e62fa3d79bc1b4b9b4aaafbd71655b3aa6397 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 1 Aug 2023 21:03:07 +0200 Subject: [PATCH 041/105] distribution: show image schema deprecation on all registries When we added this deprecation warning, some registries had not yet moved away from the deprecated specification, so we made the warning conditional for pulling from Docker Hub. That condition was added in 647dfe99a50badd27f0508c67eddc4b4923fcef7, which is over 4 Years ago, which should be time enough for images and registries to have moved to current specifications. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 8c4af5dacb2232f86c33defc0d45fcdc8af80ea0) Signed-off-by: Sebastiaan van Stijn --- distribution/pull_v2.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index 2277d4e58fdc4..b83e8defd2b28 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -438,13 +438,9 @@ func (p *puller) pullTag(ctx context.Context, ref reference.Named, platform *oci switch v := manifest.(type) { case *schema1.SignedManifest: - // give registries time to upgrade to schema2 and only warn if we know a registry has been upgraded long time ago - // TODO: condition to be removed - if reference.Domain(ref) == "docker.io" { - msg := fmt.Sprintf("[DEPRECATION NOTICE] Docker Image Format v1, and Docker Image manifest version 2, schema 1 support will be removed in an upcoming release. Suggest the author of %s to upgrade the image to the OCI Format, or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/", ref) - logrus.Warn(msg) - progress.Message(p.config.ProgressOutput, "", msg) - } + msg := fmt.Sprintf("[DEPRECATION NOTICE] Docker Image Format v1, and Docker Image manifest version 2, schema 1 support will be removed in an upcoming release. Suggest the author of %s to upgrade the image to the OCI Format, or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/", ref) + logrus.Warn(msg) + progress.Message(p.config.ProgressOutput, "", msg) id, manifestDigest, err = p.pullSchema1(ctx, ref, v, platform) if err != nil { From aaf84dd4cf88fd17f8d8af22395e4acccd667b6f Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 25 Nov 2022 10:37:05 +0100 Subject: [PATCH 042/105] remove Dockerfile.e2e Dockerfile.e2e is not used anymore. Integration tests run through the main Dockerfile. Also removes the daemon OS/Arch detection script that is not necessary anymore. It was used to select the Dockerfile based on the arch like Dockerfile.arm64 but we don't have those anymore. Was also used to check referenced frozen images in the Dockerfile. Signed-off-by: CrazyMax (cherry picked from commit 5efe72415d975ec0d4e097cfc235bd6c6c62418b) Signed-off-by: Bjorn Neergaard --- Dockerfile.e2e | 84 ------------------------- Makefile | 6 +- hack/make/.detect-daemon-osarch | 43 ------------- hack/make/.integration-daemon-setup | 7 --- hack/make/.integration-daemon-start | 4 ++ hack/make/.integration-test-helpers | 1 - hack/make/build-integration-test-binary | 7 --- hack/make/test-integration | 1 - hack/make/test-integration-cli | 6 -- hack/make/test-integration-shell | 1 - hack/test/e2e-run.sh | 1 - 11 files changed, 5 insertions(+), 156 deletions(-) delete mode 100644 Dockerfile.e2e delete mode 100644 hack/make/.detect-daemon-osarch delete mode 100644 hack/make/.integration-daemon-setup delete mode 100755 hack/make/build-integration-test-binary delete mode 100755 hack/make/test-integration-cli diff --git a/Dockerfile.e2e b/Dockerfile.e2e deleted file mode 100644 index 2b41a7e65e4de..0000000000000 --- a/Dockerfile.e2e +++ /dev/null @@ -1,84 +0,0 @@ -ARG GO_VERSION=1.20.4 - -FROM golang:${GO_VERSION}-alpine AS base -ENV GO111MODULE=off -RUN apk --no-cache add \ - bash \ - build-base \ - curl \ - lvm2-dev \ - jq - -RUN mkdir -p /build/ -RUN mkdir -p /go/src/github.com/docker/docker/ -WORKDIR /go/src/github.com/docker/docker/ - -FROM base AS frozen-images -# Get useful and necessary Hub images so we can "docker load" locally instead of pulling -COPY contrib/download-frozen-image-v2.sh / -RUN /download-frozen-image-v2.sh /build \ - busybox:latest@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 \ - busybox:latest@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 \ - debian:bullseye-slim@sha256:dacf278785a4daa9de07596ec739dbc07131e189942772210709c5c0777e8437 \ - hello-world:latest@sha256:d58e752213a51785838f9eed2b7a498ffa1cb3aa7f946dda11af39286c3db9a9 \ - arm32v7/hello-world:latest@sha256:50b8560ad574c779908da71f7ce370c0a2471c098d44d1c8f6b513c5a55eeeb1 -# See also frozenImages in "testutil/environment/protect.go" (which needs to be updated when adding images to this list) - -FROM base AS dockercli -COPY hack/dockerfile/install/install.sh ./install.sh -COPY hack/dockerfile/install/dockercli.installer ./ -RUN PREFIX=/build ./install.sh dockercli - -# TestDockerCLIBuildSuite dependency -FROM base AS contrib -COPY contrib/syscall-test /build/syscall-test -COPY contrib/httpserver/Dockerfile /build/httpserver/Dockerfile -COPY contrib/httpserver contrib/httpserver -RUN CGO_ENABLED=0 go build -buildmode=pie -o /build/httpserver/httpserver github.com/docker/docker/contrib/httpserver - -# Build the integration tests and copy the resulting binaries to /build/tests -FROM base AS builder - -# Set tag and add sources -COPY . . -# Copy test sources tests that use assert can print errors -RUN mkdir -p /build${PWD} && find integration integration-cli -name \*_test.go -exec cp --parents '{}' /build${PWD} \; -# Build and install test binaries -ARG DOCKER_GITCOMMIT=undefined -RUN hack/make.sh build-integration-test-binary -RUN mkdir -p /build/tests && find . -name test.main -exec cp --parents '{}' /build/tests \; - -## Generate testing image -FROM alpine:3.10 as runner - -ENV DOCKER_REMOTE_DAEMON=1 -ENV DOCKER_INTEGRATION_DAEMON_DEST=/ -ENTRYPOINT ["/scripts/run.sh"] - -# Add an unprivileged user to be used for tests which need it -RUN addgroup docker && adduser -D -G docker unprivilegeduser -s /bin/ash - -# GNU tar is used for generating the emptyfs image -RUN apk --no-cache add \ - bash \ - ca-certificates \ - g++ \ - git \ - inetutils-ping \ - iptables \ - libcap2-bin \ - pigz \ - tar \ - xz - -COPY hack/test/e2e-run.sh /scripts/run.sh -COPY hack/make/.build-empty-images /scripts/build-empty-images.sh - -COPY integration/testdata /tests/integration/testdata -COPY integration/build/testdata /tests/integration/build/testdata -COPY integration-cli/fixtures /tests/integration-cli/fixtures - -COPY --from=frozen-images /build/ /docker-frozen-images -COPY --from=dockercli /build/ /usr/bin/ -COPY --from=contrib /build/ /tests/contrib/ -COPY --from=builder /build/ / diff --git a/Makefile b/Makefile index ff7f713466de7..789bfc2b4eea7 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,6 @@ BUILDX ?= $(DOCKER) buildx DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) export DOCKER_GRAPHDRIVER -# get OS/Arch of docker engine -DOCKER_OSARCH := $(shell bash -c 'source hack/make/.detect-daemon-osarch && echo $${DOCKER_ENGINE_OSARCH}') -DOCKERFILE := $(shell bash -c 'source hack/make/.detect-daemon-osarch && echo $${DOCKERFILE}') - DOCKER_GITCOMMIT := $(shell git rev-parse --short HEAD || echo unsupported) export DOCKER_GITCOMMIT @@ -150,7 +146,7 @@ ifdef DOCKER_SYSTEMD DOCKER_BUILD_ARGS += --build-arg=SYSTEMD=true endif -BUILD_OPTS := ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -f "$(DOCKERFILE)" +BUILD_OPTS := ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} BUILD_CMD := $(BUILDX) build BAKE_CMD := $(BUILDX) bake diff --git a/hack/make/.detect-daemon-osarch b/hack/make/.detect-daemon-osarch deleted file mode 100644 index 9190cd0264bae..0000000000000 --- a/hack/make/.detect-daemon-osarch +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -e - -docker-version-osarch() { - if ! type docker &> /dev/null; then - # docker is not installed - return - fi - local target="$1" # "Client" or "Server" - local fmtStr="{{.${target}.Os}}/{{.${target}.Arch}}" - if docker version -f "$fmtStr" 2> /dev/null; then - # if "docker version -f" works, let's just use that! - return - fi - docker version | awk ' - $1 ~ /^(Client|Server):$/ { section = 0 } - $1 == "'"$target"':" { section = 1; next } - section && $1 == "OS/Arch:" { print $2 } - - # old versions of Docker - $1 == "OS/Arch" && $2 == "('"${target,,}"'):" { print $3 } - ' -} - -# Retrieve OS/ARCH of docker daemon, e.g. linux/amd64 -export DOCKER_ENGINE_OSARCH="${DOCKER_ENGINE_OSARCH:=$(docker-version-osarch 'Server')}" -export DOCKER_ENGINE_GOOS="${DOCKER_ENGINE_OSARCH%/*}" -export DOCKER_ENGINE_GOARCH="${DOCKER_ENGINE_OSARCH##*/}" -DOCKER_ENGINE_GOARCH=${DOCKER_ENGINE_GOARCH:=amd64} - -# and the client, just in case -export DOCKER_CLIENT_OSARCH="$(docker-version-osarch 'Client')" -export DOCKER_CLIENT_GOOS="${DOCKER_CLIENT_OSARCH%/*}" -export DOCKER_CLIENT_GOARCH="${DOCKER_CLIENT_OSARCH##*/}" -DOCKER_CLIENT_GOARCH=${DOCKER_CLIENT_GOARCH:=amd64} - -DOCKERFILE='Dockerfile' - -if [ "${DOCKER_ENGINE_GOOS:-$DOCKER_CLIENT_GOOS}" = "windows" ]; then - DOCKERFILE='Dockerfile.windows' -fi - -export DOCKERFILE diff --git a/hack/make/.integration-daemon-setup b/hack/make/.integration-daemon-setup deleted file mode 100644 index 4bcc816c2c3e8..0000000000000 --- a/hack/make/.integration-daemon-setup +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -e - -source "$MAKEDIR/.detect-daemon-osarch" -if [ "$DOCKER_ENGINE_GOOS" != "windows" ]; then - bundle .build-empty-images -fi diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index 89e38993dbbe1..04934d7b5a8b1 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -151,3 +151,7 @@ while ! ${TEST_CLIENT_BINARY} version &> /dev/null; do sleep 2 done printf "\n" + +if [ "$(docker version --format '{{ .Server.Os }}')" != 'windows' ]; then + bundle .build-empty-images +fi diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index e9062ac66cdec..16bab622c104b 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -181,7 +181,6 @@ test_env() { DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \ DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \ DOCKER_ROOTLESS="$DOCKER_ROOTLESS" \ - DOCKERFILE="$DOCKERFILE" \ GITHUB_ACTIONS="$GITHUB_ACTIONS" \ GOCACHE="$GOCACHE" \ GOPATH="$GOPATH" \ diff --git a/hack/make/build-integration-test-binary b/hack/make/build-integration-test-binary deleted file mode 100755 index 698717f0f5640..0000000000000 --- a/hack/make/build-integration-test-binary +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -# required by https://github.com/AkihiroSuda/kube-moby-integration -set -e - -source hack/make/.integration-test-helpers - -build_test_suite_binaries diff --git a/hack/make/test-integration b/hack/make/test-integration index 199131c209ca2..5cfbc89697c4d 100755 --- a/hack/make/test-integration +++ b/hack/make/test-integration @@ -12,7 +12,6 @@ fi env build_test_suite_binaries bundle .integration-daemon-start - bundle .integration-daemon-setup testexit=0 (repeat run_test_integration) || testexit=$? diff --git a/hack/make/test-integration-cli b/hack/make/test-integration-cli deleted file mode 100755 index 480851e70f487..0000000000000 --- a/hack/make/test-integration-cli +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e -echo "WARNING: test-integration-cli is DEPRECATED. Use test-integration." >&2 - -# TODO: remove this and exit 1 once CI has changed to use test-integration -bundle test-integration diff --git a/hack/make/test-integration-shell b/hack/make/test-integration-shell index bcfa4682eb324..1ee23b3806054 100644 --- a/hack/make/test-integration-shell +++ b/hack/make/test-integration-shell @@ -1,7 +1,6 @@ #!/usr/bin/env bash bundle .integration-daemon-start -bundle .integration-daemon-setup export ABS_DEST bash +e diff --git a/hack/test/e2e-run.sh b/hack/test/e2e-run.sh index 545504fa0e7c8..5e4d8f2154b88 100755 --- a/hack/test/e2e-run.sh +++ b/hack/test/e2e-run.sh @@ -59,7 +59,6 @@ test_env() { DOCKER_HOST="$DOCKER_HOST" \ DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \ DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \ - DOCKERFILE="$DOCKERFILE" \ GOPATH="$GOPATH" \ GOTRACEBACK=all \ HOME="$ABS_DEST/fake-HOME" \ From 5eef5a7f59a069152f3debfd8a55f9c0b078d253 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 18 Aug 2023 08:00:02 -0600 Subject: [PATCH 043/105] ci(bin-image): clean up env var handling There are still messy special cases (e.g. DOCKER_GITCOMMIT vs VERSION), but this makes things a little easier to follow, as we keep GHA-specifics in the GHA files. Signed-off-by: Bjorn Neergaard (cherry picked from commit ad91fc1b0014a68efc660243288f4e65111794de) Signed-off-by: Bjorn Neergaard --- .github/workflows/bin-image.yml | 4 ++-- docker-bake.hcl | 16 ++-------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index 19286d226a90c..b5fce120bf0aa 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -16,6 +16,8 @@ on: env: MOBYBIN_REPO_SLUG: moby/moby-bin + DOCKER_GITCOMMIT: ${{ github.sha }} + VERSION: ${{ github.ref }} PLATFORM: Moby Engine PRODUCT: Moby DEFAULT_PRODUCT_LICENSE: Moby @@ -113,8 +115,6 @@ jobs: name: Build id: bake uses: docker/bake-action@v3 - env: - DOCKER_GITCOMMIT: ${{ github.sha }} with: files: | ./docker-bake.hcl diff --git a/docker-bake.hcl b/docker-bake.hcl index a1ac477a91b08..5666f83d94762 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -47,18 +47,6 @@ variable "PACKAGER_NAME" { default = "" } -# GITHUB_REF is the actual ref that triggers the workflow and used as version -# when tag is pushed: https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables -variable "GITHUB_REF" { - default = "" -} - -# GITHUB_SHA is the commit SHA that triggered the workflow and used as commit. -# https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables -variable "GITHUB_SHA" { - default = "" -} - # Special target: https://github.com/docker/metadata-action#bake-definition target "docker-metadata-action" { tags = ["moby-bin:local"] @@ -81,8 +69,8 @@ target "_common" { DOCKER_STATIC = DOCKER_STATIC DOCKER_LDFLAGS = DOCKER_LDFLAGS DOCKER_BUILDTAGS = DOCKER_BUILDTAGS - DOCKER_GITCOMMIT = DOCKER_GITCOMMIT != null ? DOCKER_GITCOMMIT : GITHUB_SHA - VERSION = VERSION != "" ? VERSION : GITHUB_REF + DOCKER_GITCOMMIT = DOCKER_GITCOMMIT + VERSION = VERSION PLATFORM = PLATFORM PRODUCT = PRODUCT DEFAULT_PRODUCT_LICENSE = DEFAULT_PRODUCT_LICENSE From ac2a80fcc3ef348b20dfb77f2703543d9ee4c7a2 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 18 Aug 2023 08:09:55 -0600 Subject: [PATCH 044/105] ci(bin-image): clean up metadata Signed-off-by: Bjorn Neergaard (cherry picked from commit 2010f4338e2054177b364e727878bfadd2e68d5c) Signed-off-by: Bjorn Neergaard --- .github/workflows/bin-image.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml index b5fce120bf0aa..17688ea652132 100644 --- a/.github/workflows/bin-image.yml +++ b/.github/workflows/bin-image.yml @@ -18,10 +18,9 @@ env: MOBYBIN_REPO_SLUG: moby/moby-bin DOCKER_GITCOMMIT: ${{ github.sha }} VERSION: ${{ github.ref }} - PLATFORM: Moby Engine - PRODUCT: Moby - DEFAULT_PRODUCT_LICENSE: Moby - PACKAGER_NAME: Moby + PLATFORM: Moby Engine - Nightly + PRODUCT: moby-bin + PACKAGER_NAME: The Moby Project jobs: validate-dco: From 4ac2355d6274fe195e9b9f93200b362e48f91713 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 18 Aug 2023 08:15:30 -0600 Subject: [PATCH 045/105] hack: use long SHA for DOCKER_GITCOMMIT This better aligns to GHA/CI settings, and is in general a better practice in the year 2023. We also drop the 'unsupported' fallback for `git rev-parse` in the Makefile; we have a better fallback behavior for an empty DOCKER_GITCOMMIT in `hack/make.sh`. Signed-off-by: Bjorn Neergaard (cherry picked from commit d125823d3fef096013912353f723b620f34d038a) Signed-off-by: Bjorn Neergaard --- Makefile | 2 +- hack/make.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 789bfc2b4eea7..8693ea307981e 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ BUILDX ?= $(DOCKER) buildx DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) export DOCKER_GRAPHDRIVER -DOCKER_GITCOMMIT := $(shell git rev-parse --short HEAD || echo unsupported) +DOCKER_GITCOMMIT := $(shell git rev-parse HEAD) export DOCKER_GITCOMMIT # allow overriding the repository and branch that validation scripts are running diff --git a/hack/make.sh b/hack/make.sh index 189ee5aa8b170..d8c7667cda4be 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -51,7 +51,7 @@ fi if [ "$DOCKER_GITCOMMIT" ]; then GITCOMMIT="$DOCKER_GITCOMMIT" elif command -v git &> /dev/null && [ -e .git ] && git rev-parse &> /dev/null; then - GITCOMMIT=$(git rev-parse --short HEAD) + GITCOMMIT=$(git rev-parse HEAD) if [ -n "$(git status --porcelain --untracked-files=no)" ]; then GITCOMMIT="$GITCOMMIT-unsupported" echo "#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -66,8 +66,8 @@ elif command -v git &> /dev/null && [ -e .git ] && git rev-parse &> /dev/null; t else echo >&2 'error: .git directory missing and DOCKER_GITCOMMIT not specified' echo >&2 ' Please either build with the .git directory accessible, or specify the' - echo >&2 ' exact (--short) commit hash you are building using DOCKER_GITCOMMIT for' - echo >&2 ' future accountability in diagnosing build issues. Thanks!' + echo >&2 ' exact commit hash you are building using DOCKER_GITCOMMIT for future' + echo >&2 ' accountability in diagnosing build issues. Thanks!' exit 1 fi From b83f5a89f481ab5b06816932cb8b2b09e5f62c50 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Tue, 22 Aug 2023 11:45:27 +0200 Subject: [PATCH 046/105] Don't return an error if the lease is not found If the image for the wanted platform doesn't exist then the lease doesn't exist either. Returning this error hides the real error, so let's not return it. Signed-off-by: Djordje Lukic (cherry picked from commit b8ff8ea58ee37d672f42e94da4d73442d8a81fc9) Signed-off-by: Sebastiaan van Stijn --- daemon/images/image.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/daemon/images/image.go b/daemon/images/image.go index a773bd0cd2cbc..553f3c46dd50b 100644 --- a/daemon/images/image.go +++ b/daemon/images/image.go @@ -54,10 +54,13 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, image str func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.Image, platform ocispec.Platform) (bool, error) { logger := logrus.WithField("image", img.ID).WithField("desiredPlatform", platforms.Format(platform)) - ls, leaseErr := i.leases.ListResources(ctx, leases.Lease{ID: imageKey(img.ID().String())}) - if leaseErr != nil { - logger.WithError(leaseErr).Error("Error looking up image leases") - return false, leaseErr + ls, err := i.leases.ListResources(ctx, leases.Lease{ID: imageKey(img.ID().String())}) + if err != nil { + if cerrdefs.IsNotFound(err) { + return false, nil + } + logger.WithError(err).Error("Error looking up image leases") + return false, err } // Note we are comparing against manifest lists here, which we expect to always have a CPU variant set (where applicable). From 49671250f6d6f91537a0decb44164b4b34d74b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 21 Aug 2023 16:50:07 +0200 Subject: [PATCH 047/105] c8d/commit: Don't produce an empty layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the diff is empty and don't produce an empty layer. Signed-off-by: Paweł Gronowski (cherry picked from commit eb56493f4ea0fe3e02f3dbf59631cb1a13aa5c63) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/image_commit.go | 89 ++++++++++++++++++++----------- pkg/archive/diff.go | 19 +++++++ 2 files changed, 76 insertions(+), 32 deletions(-) diff --git a/daemon/containerd/image_commit.go b/daemon/containerd/image_commit.go index 56941ee58625c..077b53efa3c21 100644 --- a/daemon/containerd/image_commit.go +++ b/daemon/containerd/image_commit.go @@ -20,6 +20,7 @@ import ( "github.com/containerd/containerd/snapshots" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/image" + "github.com/docker/docker/pkg/archive" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" "github.com/opencontainers/image-spec/specs-go" @@ -38,28 +39,27 @@ func (i *ImageService) CommitImage(ctx context.Context, cc backend.CommitConfig) container := i.containers.Get(cc.ContainerID) cs := i.client.ContentStore() - imageManifest, err := getContainerImageManifest(container) - if err != nil { - return "", err - } + var parentManifest ocispec.Manifest + var parentImage ocispec.Image - imageManifestBytes, err := content.ReadBlob(ctx, cs, imageManifest) - if err != nil { - return "", err - } + // ImageManifest can be nil when committing an image with base FROM scratch + if container.ImageManifest != nil { + imageManifestBytes, err := content.ReadBlob(ctx, cs, *container.ImageManifest) + if err != nil { + return "", err + } - var manifest ocispec.Manifest - if err := json.Unmarshal(imageManifestBytes, &manifest); err != nil { - return "", err - } + if err := json.Unmarshal(imageManifestBytes, &parentManifest); err != nil { + return "", err + } - imageConfigBytes, err := content.ReadBlob(ctx, cs, manifest.Config) - if err != nil { - return "", err - } - var ociimage ocispec.Image - if err := json.Unmarshal(imageConfigBytes, &ociimage); err != nil { - return "", err + imageConfigBytes, err := content.ReadBlob(ctx, cs, parentManifest.Config) + if err != nil { + return "", err + } + if err := json.Unmarshal(imageConfigBytes, &parentImage); err != nil { + return "", err + } } var ( @@ -78,15 +78,19 @@ func (i *ImageService) CommitImage(ctx context.Context, cc backend.CommitConfig) if err != nil { return "", fmt.Errorf("failed to export layer: %w", err) } + imageConfig := generateCommitImageConfig(parentImage, diffID, cc) + + layers := parentManifest.Layers + if diffLayerDesc != nil { + rootfsID := identity.ChainID(imageConfig.RootFS.DiffIDs).String() - imageConfig := generateCommitImageConfig(ociimage, diffID, cc) + if err := applyDiffLayer(ctx, rootfsID, parentImage, sn, differ, *diffLayerDesc); err != nil { + return "", fmt.Errorf("failed to apply diff: %w", err) + } - rootfsID := identity.ChainID(imageConfig.RootFS.DiffIDs).String() - if err := applyDiffLayer(ctx, rootfsID, ociimage, sn, differ, diffLayerDesc); err != nil { - return "", fmt.Errorf("failed to apply diff: %w", err) + layers = append(layers, *diffLayerDesc) } - layers := append(manifest.Layers, diffLayerDesc) commitManifestDesc, err := writeContentsForImage(ctx, container.Driver, cs, imageConfig, layers) if err != nil { return "", err @@ -130,6 +134,12 @@ func generateCommitImageConfig(baseConfig ocispec.Image, diffID digest.Digest, o logrus.Warnf("assuming os=%q", os) } logrus.Debugf("generateCommitImageConfig(): arch=%q, os=%q", arch, os) + + diffIds := baseConfig.RootFS.DiffIDs + if diffID != "" { + diffIds = append(diffIds, diffID) + } + return ocispec.Image{ Platform: ocispec.Platform{ Architecture: arch, @@ -140,7 +150,7 @@ func generateCommitImageConfig(baseConfig ocispec.Image, diffID digest.Digest, o Config: containerConfigToOciImageConfig(opts.Config), RootFS: ocispec.RootFS{ Type: "layers", - DiffIDs: append(baseConfig.RootFS.DiffIDs, diffID), + DiffIDs: diffIds, }, History: append(baseConfig.History, ocispec.History{ Created: &createdTime, @@ -217,28 +227,43 @@ func writeContentsForImage(ctx context.Context, snName string, cs content.Store, } // createDiff creates a layer diff into containerd's content store. -func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs content.Store, comparer diff.Comparer) (ocispec.Descriptor, digest.Digest, error) { +// If the diff is empty it returns nil empty digest and no error. +func createDiff(ctx context.Context, name string, sn snapshots.Snapshotter, cs content.Store, comparer diff.Comparer) (*ocispec.Descriptor, digest.Digest, error) { newDesc, err := rootfs.CreateDiff(ctx, name, sn, comparer) if err != nil { - return ocispec.Descriptor{}, "", err + return nil, "", err + } + + ra, err := cs.ReaderAt(ctx, newDesc) + if err != nil { + return nil, "", fmt.Errorf("failed to read diff archive: %w", err) + } + defer ra.Close() + + empty, err := archive.IsEmpty(content.NewReader(ra)) + if err != nil { + return nil, "", fmt.Errorf("failed to check if archive is empty: %w", err) + } + if empty { + return nil, "", nil } info, err := cs.Info(ctx, newDesc.Digest) if err != nil { - return ocispec.Descriptor{}, "", err + return nil, "", fmt.Errorf("failed to get content info: %w", err) } diffIDStr, ok := info.Labels["containerd.io/uncompressed"] if !ok { - return ocispec.Descriptor{}, "", fmt.Errorf("invalid differ response with no diffID") + return nil, "", fmt.Errorf("invalid differ response with no diffID") } diffID, err := digest.Parse(diffIDStr) if err != nil { - return ocispec.Descriptor{}, "", err + return nil, "", err } - return ocispec.Descriptor{ + return &ocispec.Descriptor{ MediaType: ocispec.MediaTypeImageLayerGzip, Digest: newDesc.Digest, Size: info.Size, @@ -254,7 +279,7 @@ func applyDiffLayer(ctx context.Context, name string, baseImg ocispec.Image, sn mount, err := sn.Prepare(ctx, key, parent) if err != nil { - return err + return fmt.Errorf("failed to prepare snapshot: %w", err) } defer func() { diff --git a/pkg/archive/diff.go b/pkg/archive/diff.go index c8c7be74797e8..1a2fb971f979c 100644 --- a/pkg/archive/diff.go +++ b/pkg/archive/diff.go @@ -223,6 +223,25 @@ func ApplyUncompressedLayer(dest string, layer io.Reader, options *TarOptions) ( return applyLayerHandler(dest, layer, options, false) } +// IsEmpty checks if the tar archive is empty (doesn't contain any entries). +func IsEmpty(rd io.Reader) (bool, error) { + decompRd, err := DecompressStream(rd) + if err != nil { + return true, fmt.Errorf("failed to decompress archive: %v", err) + } + defer decompRd.Close() + + tarReader := tar.NewReader(decompRd) + if _, err := tarReader.Next(); err != nil { + if err == io.EOF { + return true, nil + } + return false, fmt.Errorf("failed to read next archive header: %v", err) + } + + return false, nil +} + // do the bulk load of ApplyLayer, but allow for not calling DecompressStream func applyLayerHandler(dest string, layer io.Reader, options *TarOptions, decompress bool) (int64, error) { dest = filepath.Clean(dest) From 63422515ba0ef223ae78f5d9a7a09b08855f72cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 21 Aug 2023 16:50:49 +0200 Subject: [PATCH 048/105] c8d/run: Allow running container without image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows the legacy builder to apply changes to the `FROM scratch` layer. Signed-off-by: Paweł Gronowski (cherry picked from commit dfaff9598cf9207aaf2e275248d9cf8e8b9d3b54) Signed-off-by: Sebastiaan van Stijn --- daemon/containerd/image_snapshot.go | 57 +++++++++++++++-------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/daemon/containerd/image_snapshot.go b/daemon/containerd/image_snapshot.go index 962c2100dc223..21e36a4f7c75b 100644 --- a/daemon/containerd/image_snapshot.go +++ b/daemon/containerd/image_snapshot.go @@ -18,42 +18,45 @@ import ( // PrepareSnapshot prepares a snapshot from a parent image for a container func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *ocispec.Platform) error { - img, err := i.resolveImage(ctx, parentImage) - if err != nil { - return err - } + var parentSnapshot string + if parentImage != "" { + img, err := i.resolveImage(ctx, parentImage) + if err != nil { + return err + } - cs := i.client.ContentStore() + cs := i.client.ContentStore() - matcher := platforms.Default() - if platform != nil { - matcher = platforms.Only(*platform) - } + matcher := platforms.Default() + if platform != nil { + matcher = platforms.Only(*platform) + } - platformImg := containerd.NewImageWithPlatform(i.client, img, matcher) - unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter) - if err != nil { - return err - } + platformImg := containerd.NewImageWithPlatform(i.client, img, matcher) + unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter) + if err != nil { + return err + } - if !unpacked { - if err := platformImg.Unpack(ctx, i.snapshotter); err != nil { + if !unpacked { + if err := platformImg.Unpack(ctx, i.snapshotter); err != nil { + return err + } + } + + desc, err := containerdimages.Config(ctx, cs, img.Target, matcher) + if err != nil { return err } - } - desc, err := containerdimages.Config(ctx, cs, img.Target, matcher) - if err != nil { - return err - } + diffIDs, err := containerdimages.RootFS(ctx, cs, desc) + if err != nil { + return err + } - diffIDs, err := containerdimages.RootFS(ctx, cs, desc) - if err != nil { - return err + parentSnapshot = identity.ChainID(diffIDs).String() } - parent := identity.ChainID(diffIDs).String() - // Add a lease so that containerd doesn't garbage collect our snapshot ls := i.client.LeasesService() lease, err := ls.Create(ctx, leases.WithID(id)) @@ -69,7 +72,7 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentIma } s := i.client.SnapshotService(i.StorageDriver()) - _, err = s.Prepare(ctx, id, parent) + _, err = s.Prepare(ctx, id, parentSnapshot) return err } From 1d10e8633d5876517c689b788e39533370cb2c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 21 Aug 2023 16:52:32 +0200 Subject: [PATCH 049/105] daemon: Handle NotFound when deleting container lease MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the lease doesn't exit (for example when creating the container failed), just ignore the not found error. Signed-off-by: Paweł Gronowski (cherry picked from commit bedcc94de4cab9f3072a98f71806bc18e47cd072) Signed-off-by: Sebastiaan van Stijn --- daemon/delete.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/daemon/delete.go b/daemon/delete.go index 80cddd91a741d..bfd974e5b413c 100644 --- a/daemon/delete.go +++ b/daemon/delete.go @@ -8,6 +8,7 @@ import ( "strings" "time" + cerrdefs "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/leases" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" @@ -144,8 +145,10 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, config ty ID: container.ID, } if err := ls.Delete(context.Background(), lease, leases.SynchronousDelete); err != nil { - container.SetRemovalError(err) - return err + if !cerrdefs.IsNotFound(err) { + container.SetRemovalError(err) + return err + } } } } From ed2f5d1d85bfd976652b9879be20fcf8a33fbe76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 23 Aug 2023 13:53:46 +0200 Subject: [PATCH 050/105] c8d/builder: Don't drop fields from created image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous image created a new partially filled image. This caused child images to lose their parent's layers. Instead of creating a new object and trying to replace its fields, just clone the original passed image and change its ID to the manifest digest. Signed-off-by: Paweł Gronowski (cherry picked from commit 01214bafd211d1f8c74c1147c39338b27372ede5) Signed-off-by: Paweł Gronowski --- daemon/containerd/image_builder.go | 5 +---- image/image.go | 9 +++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/daemon/containerd/image_builder.go b/daemon/containerd/image_builder.go index 05338da2dada5..fb864acd05855 100644 --- a/daemon/containerd/image_builder.go +++ b/daemon/containerd/image_builder.go @@ -505,9 +505,6 @@ func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent st return nil, err } - newImage := dimage.NewImage(dimage.ID(createdImage.Target.Digest)) - newImage.V1Image = imgToCreate.V1Image - newImage.V1Image.ID = string(createdImage.Target.Digest) - newImage.History = imgToCreate.History + newImage := dimage.Clone(imgToCreate, dimage.ID(createdImage.Target.Digest)) return newImage, nil } diff --git a/image/image.go b/image/image.go index 856a845b17f98..d17601323ad0f 100644 --- a/image/image.go +++ b/image/image.go @@ -248,6 +248,15 @@ func NewChildImage(img *Image, child ChildConfig, os string) *Image { } } +// Clone clones an image and changes ID. +func Clone(base *Image, id ID) *Image { + img := *base + img.RootFS = img.RootFS.Clone() + img.V1Image.ID = id.String() + img.computedID = id + return &img +} + // History stores build commands that were used to create an image type History struct { // Created is the timestamp at which the image was created From 088cec8f0f592a01083d6fcf085e1a857860e6bd Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:30:15 +0200 Subject: [PATCH 051/105] hack: update link to GOPATH documentation This documentation moved to a different page, and the Go documentation moved to the https://go.dev/ domain. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 2aabd6447753bde5c844b2549cd2330cb1383bb6) Signed-off-by: Sebastiaan van Stijn --- hack/make.ps1 | 2 +- hack/make.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/make.ps1 b/hack/make.ps1 index 82a38f3cfce3c..5c9d3a951d34f 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -459,7 +459,7 @@ Try { if (-not $inContainer) { Verify-GoVersion } # Verify GOPATH is set - if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://golang.org/doc/code.html#GOPATH" } + if ($env:GOPATH.Length -eq 0) { Throw "Missing GOPATH environment variable. See https://pkg.go.dev/cmd/go#hdr-GOPATH_environment_variable" } # Run autogen if building daemon. if ($Daemon) { diff --git a/hack/make.sh b/hack/make.sh index d8c7667cda4be..ec01bc1068cb5 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -79,7 +79,7 @@ if [ "$AUTO_GOPATH" ]; then fi if [ ! "$GOPATH" ]; then - echo >&2 'error: missing GOPATH; please see https://golang.org/doc/code.html#GOPATH' + echo >&2 'error: missing GOPATH; please see https://pkg.go.dev/cmd/go#hdr-GOPATH_environment_variable' echo >&2 ' alternatively, set AUTO_GOPATH=1' exit 1 fi From 377af4c9b4ee16f178a7899fdf953a99ad8daf06 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:45:50 +0200 Subject: [PATCH 052/105] Dockerfile: Windows: update Golang download domains to cut down redirects The `golang.org` domain moved to `go.dev`, and the download-URLs we were using resulted in 2 redirects; curl -sI https://golang.org/dl/go1.20.windows-amd64.zip | grep 'location' location: https://go.dev/dl/go1.20.windows-amd64.zip curl -sI https://go.dev/dl/go1.20.windows-amd64.zip | grep 'location' location: https://dl.google.com/go/go1.20.windows-amd64.zip curl -sI https://dl.google.com/go/go1.20.windows-amd64.zip HTTP/2 200 # ... This patch cuts it down to one redirects. I decided not to use the "final" (`dl.google.com`) URL, because that URL is not documented in the Golang docs, and visiting the domain itself (https://dl.google.com/) redirects to a marketing page for "Google Chrome". Trying the `/go/` path (https://dl.google.com/go/) also does not show a landing page that lists downloads, so I'm considering those URLs to be "unstable". Signed-off-by: Sebastiaan van Stijn (cherry picked from commit f6a5318f9411ae26bb6d7335b36505c83bbc21b9) Signed-off-by: Sebastiaan van Stijn --- Dockerfile.windows | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index d3783a9331bf7..2f20f52112413 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -224,7 +224,7 @@ RUN ` ` Write-Host INFO: Downloading go...; ` $dlGoVersion=$Env:GO_VERSION -replace '\.0$',''; ` - Download-File "https://golang.org/dl/go${dlGoVersion}.windows-amd64.zip" C:\go.zip; ` + Download-File "https://go.dev/dl/go${dlGoVersion}.windows-amd64.zip" C:\go.zip; ` ` Write-Host INFO: Downloading compiler 1 of 3...; ` Download-File https://raw.githubusercontent.com/moby/docker-tdmgcc/master/gcc.zip C:\gcc.zip; ` From de13951b9d0fb61595aeec48a14d534cb2fdad07 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:21:50 +0200 Subject: [PATCH 053/105] docs/api: update links to Go documentation Go documentation moved to the `go.dev` domain; curl -sI https://golang.org/doc/install/source#environment | grep 'location' location: https://go.dev/doc/install/source Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 4862d391442df10caca6f57104876deb754296d1) Signed-off-by: Sebastiaan van Stijn --- docs/api/v1.32.yaml | 4 ++-- docs/api/v1.33.yaml | 4 ++-- docs/api/v1.34.yaml | 4 ++-- docs/api/v1.35.yaml | 4 ++-- docs/api/v1.36.yaml | 4 ++-- docs/api/v1.37.yaml | 4 ++-- docs/api/v1.38.yaml | 4 ++-- docs/api/v1.39.yaml | 4 ++-- docs/api/v1.40.yaml | 4 ++-- docs/api/v1.41.yaml | 4 ++-- docs/api/v1.42.yaml | 4 ++-- docs/api/v1.43.yaml | 4 ++-- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/api/v1.32.yaml b/docs/api/v1.32.yaml index b843bc63406c4..ca9178c634f98 100644 --- a/docs/api/v1.32.yaml +++ b/docs/api/v1.32.yaml @@ -3632,7 +3632,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3640,7 +3640,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.33.yaml b/docs/api/v1.33.yaml index 3fa7eb43d5c58..fbf8476a13788 100644 --- a/docs/api/v1.33.yaml +++ b/docs/api/v1.33.yaml @@ -3637,7 +3637,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3645,7 +3645,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.34.yaml b/docs/api/v1.34.yaml index d90d4dbbf84d4..42cff6187d901 100644 --- a/docs/api/v1.34.yaml +++ b/docs/api/v1.34.yaml @@ -3666,7 +3666,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3674,7 +3674,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.35.yaml b/docs/api/v1.35.yaml index a3ed1494669fe..cb1c8d3709825 100644 --- a/docs/api/v1.35.yaml +++ b/docs/api/v1.35.yaml @@ -3648,7 +3648,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3656,7 +3656,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.36.yaml b/docs/api/v1.36.yaml index d5f4da5440eb6..a656e106bd6e0 100644 --- a/docs/api/v1.36.yaml +++ b/docs/api/v1.36.yaml @@ -3661,7 +3661,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3669,7 +3669,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.37.yaml b/docs/api/v1.37.yaml index b9290e1fea1dc..29a6640583f4e 100644 --- a/docs/api/v1.37.yaml +++ b/docs/api/v1.37.yaml @@ -3681,7 +3681,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3689,7 +3689,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.38.yaml b/docs/api/v1.38.yaml index af7b11817d86e..c1473afeee5c5 100644 --- a/docs/api/v1.38.yaml +++ b/docs/api/v1.38.yaml @@ -3735,7 +3735,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -3743,7 +3743,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.39.yaml b/docs/api/v1.39.yaml index b7d7943af5b79..4102875ad8bc4 100644 --- a/docs/api/v1.39.yaml +++ b/docs/api/v1.39.yaml @@ -4719,7 +4719,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -4727,7 +4727,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.40.yaml b/docs/api/v1.40.yaml index 98df0853939e9..0d33d9908f111 100644 --- a/docs/api/v1.40.yaml +++ b/docs/api/v1.40.yaml @@ -4855,7 +4855,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -4863,7 +4863,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.41.yaml b/docs/api/v1.41.yaml index 13628c4348dd3..20cd26f5e5688 100644 --- a/docs/api/v1.41.yaml +++ b/docs/api/v1.41.yaml @@ -5006,7 +5006,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -5014,7 +5014,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.42.yaml b/docs/api/v1.42.yaml index ab1a740d30a14..a452c486b253c 100644 --- a/docs/api/v1.42.yaml +++ b/docs/api/v1.42.yaml @@ -5036,7 +5036,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -5044,7 +5044,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: diff --git a/docs/api/v1.43.yaml b/docs/api/v1.43.yaml index a616794c1e2f3..08cf7e52b674c 100644 --- a/docs/api/v1.43.yaml +++ b/docs/api/v1.43.yaml @@ -5068,7 +5068,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -5076,7 +5076,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: From 73f6053bb3183eb568a7eed6e08e071216e904bf Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:22:19 +0200 Subject: [PATCH 054/105] api: swagger: update link to Go documentation Go documentation moved to the `go.dev` domain; curl -sI https://golang.org/doc/install/source#environment | grep 'location' location: https://go.dev/doc/install/source Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 136e86bb5c08aaaf87fb8119f883133c077e3bcf) Signed-off-by: Sebastiaan van Stijn --- api/swagger.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index a820f996f94f6..7635b9f665ab2 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -5068,7 +5068,7 @@ definitions: Go runtime (`GOOS`). Currently returned values are "linux" and "windows". A full list of - possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "linux" Architecture: @@ -5076,7 +5076,7 @@ definitions: Hardware architecture of the host, as returned by the Go runtime (`GOARCH`). - A full list of possible values can be found in the [Go documentation](https://golang.org/doc/install/source#environment). + A full list of possible values can be found in the [Go documentation](https://go.dev/doc/install/source#environment). type: "string" example: "x86_64" NCPU: From d2e9a19358a2e2dd4e0beca007ed9ba489b7ea76 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 10:28:05 +0200 Subject: [PATCH 055/105] CONTRIBUTING.md: update links to golang docs and blog - docs moved to https://go.dev/doc/ - blog moved to https://go.dev/blog/ Signed-off-by: Sebastiaan van Stijn (cherry picked from commit b18e170631998d9fdea893d6b3961aecc65cebae) Signed-off-by: Sebastiaan van Stijn --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 531ac610be4b8..e3bf263a9813c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -422,6 +422,6 @@ The rules: guidelines. Since you've read all the rules, you now know that. If you are having trouble getting into the mood of idiomatic Go, we recommend -reading through [Effective Go](https://golang.org/doc/effective_go.html). The -[Go Blog](https://blog.golang.org) is also a great resource. Drinking the +reading through [Effective Go](https://go.dev/doc/effective_go). The +[Go Blog](https://go.dev/blog/) is also a great resource. Drinking the kool-aid is a lot easier than going thirsty. From 1d983e2e8acb4324a5a8df77033d35787e96efdc Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 25 Aug 2023 02:19:35 +0200 Subject: [PATCH 056/105] update golangci-lint to v1.54.2 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit cd49f9affdad81360620019b6e8741ae73fd5590) Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 2 +- daemon/logger/splunk/splunk.go | 2 +- hack/validate/golangci-lint.yml | 11 +++++------ testutil/environment/special_images.go | 12 ++++++++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e4b61fa5b031..9c3716868f036 100644 --- a/Dockerfile +++ b/Dockerfile @@ -228,7 +228,7 @@ FROM binary-dummy AS containerd-windows FROM containerd-${TARGETOS} AS containerd FROM base AS golangci_lint -ARG GOLANGCI_LINT_VERSION=v1.51.2 +ARG GOLANGCI_LINT_VERSION=v1.54.2 RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ GOBIN=/build/ GO111MODULE=on go install "github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}" \ diff --git a/daemon/logger/splunk/splunk.go b/daemon/logger/splunk/splunk.go index d194334dadc80..7b495d24c2d14 100644 --- a/daemon/logger/splunk/splunk.go +++ b/daemon/logger/splunk/splunk.go @@ -37,7 +37,7 @@ const ( splunkCANameKey = "splunk-caname" splunkInsecureSkipVerifyKey = "splunk-insecureskipverify" splunkFormatKey = "splunk-format" - splunkVerifyConnectionKey = "splunk-verify-connection" + splunkVerifyConnectionKey = "splunk-verify-connection" // #nosec G101 -- ignoring: Potential hardcoded credentials (gosec) splunkGzipCompressionKey = "splunk-gzip" splunkGzipCompressionLevelKey = "splunk-gzip-level" splunkIndexAcknowledgment = "splunk-index-acknowledgment" diff --git a/hack/validate/golangci-lint.yml b/hack/validate/golangci-lint.yml index 2832ad7e77352..89221ec2da895 100644 --- a/hack/validate/golangci-lint.yml +++ b/hack/validate/golangci-lint.yml @@ -40,12 +40,11 @@ linters-settings: govet: check-shadowing: false depguard: - list-type: blacklist - include-go-root: true - packages: - # The io/ioutil package has been deprecated. - # https://go.dev/doc/go1.16#ioutil - - io/ioutil + rules: + main: + deny: + - pkg: io/ioutil + desc: The io/ioutil package has been deprecated, see https://go.dev/doc/go1.16#ioutil revive: rules: # FIXME make sure all packages have a description. Currently, there's many packages without. diff --git a/testutil/environment/special_images.go b/testutil/environment/special_images.go index a832cd7c3c645..e50efc8743543 100644 --- a/testutil/environment/special_images.go +++ b/testutil/environment/special_images.go @@ -1,10 +1,14 @@ package environment -// Graph driver image store identifies images by the ID of their config. -const DanglingImageIdGraphDriver = "sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43" +// DanglingImageIdGraphDriver is the digest for dangling images used +// in tests when the graph driver is used. The graph driver image store +// identifies images by the ID of their config. +const DanglingImageIdGraphDriver = "sha256:0df1207206e5288f4a989a2f13d1f5b3c4e70467702c1d5d21dfc9f002b7bd43" // #nosec G101 -- ignoring: Potential hardcoded credentials (gosec) -// The containerd image store identifies images by the ID of their manifest/manifest list. -const DanglingImageIdSnapshotter = "sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456" +// DanglingImageIdSnapshotter is the digest for dangling images used in +// tests when the containerd image store is used. The container image +// store identifies images by the ID of their manifest/manifest list.. +const DanglingImageIdSnapshotter = "sha256:16d365089e5c10e1673ee82ab5bba38ade9b763296ad918bd24b42a1156c5456" // #nosec G101 -- ignoring: Potential hardcoded credentials (gosec) func GetTestDanglingImageId(testEnv *Execution) string { if testEnv.UsingSnapshotter() { From 5d4cc0b5b576dd7123c65ba3b030d194507bd829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 17 Aug 2023 16:44:22 +0200 Subject: [PATCH 057/105] integration/liveRestore: Check volume content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that the content in the live-restored volume mounted in a new container is the same as the content in the old container. This checks if volume's _data directory doesn't get unmounted on startup. Signed-off-by: Paweł Gronowski (cherry picked from commit aef703fa1b85faffb6c13db8860b3deecbb9de2f) Signed-off-by: Sebastiaan van Stijn --- integration/daemon/daemon_test.go | 52 ++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go index f4565c9616766..5ce70d8418e56 100644 --- a/integration/daemon/daemon_test.go +++ b/integration/daemon/daemon_test.go @@ -1,14 +1,17 @@ package daemon // import "github.com/docker/docker/integration/daemon" import ( + "bytes" "context" "fmt" + "io" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" + "strings" "syscall" "testing" @@ -16,7 +19,9 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/daemon/config" + "github.com/docker/docker/errdefs" "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/testutil/daemon" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -431,9 +436,28 @@ func testLiveRestoreVolumeReferences(t *testing.T) { Source: v.Name, Target: "/foo", } - cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("top")) + + const testContent = "hello" + cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("sh", "-c", "echo "+testContent+">>/foo/test.txt; sleep infinity")) defer c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) + // Wait until container creates a file in the volume. + poll.WaitOn(t, func(t poll.LogT) poll.Result { + stat, err := c.ContainerStatPath(ctx, cID, "/foo/test.txt") + if err != nil { + if errdefs.IsNotFound(err) { + return poll.Continue("file doesn't yet exist") + } + return poll.Error(err) + } + + if int(stat.Size) != len(testContent)+1 { + return poll.Error(fmt.Errorf("unexpected test file size: %d", stat.Size)) + } + + return poll.Success() + }) + d.Restart(t, "--live-restore", "--iptables=false") // Try to remove the volume @@ -441,6 +465,32 @@ func testLiveRestoreVolumeReferences(t *testing.T) { err = c.VolumeRemove(ctx, v.Name, false) assert.ErrorContains(t, err, "volume is in use") + t.Run("volume still mounted", func(t *testing.T) { + skip.If(t, testEnv.IsRootless(), "restarted rootless daemon has a new mount namespace and it won't have the previous mounts") + + // Check if a new container with the same volume has access to the previous content. + // This fails if the volume gets unmounted at startup. + cID2 := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("cat", "/foo/test.txt")) + defer c.ContainerRemove(ctx, cID2, types.ContainerRemoveOptions{Force: true}) + + poll.WaitOn(t, container.IsStopped(ctx, c, cID2)) + + inspect, err := c.ContainerInspect(ctx, cID2) + if assert.Check(t, err) { + assert.Check(t, is.Equal(inspect.State.ExitCode, 0), "volume doesn't have the same file") + } + + logs, err := c.ContainerLogs(ctx, cID2, types.ContainerLogsOptions{ShowStdout: true}) + assert.NilError(t, err) + defer logs.Close() + + var stdoutBuf bytes.Buffer + _, err = stdcopy.StdCopy(&stdoutBuf, io.Discard, logs) + assert.NilError(t, err) + + assert.Check(t, is.Equal(strings.TrimSpace(stdoutBuf.String()), testContent)) + }) + // Remove that container which should free the references in the volume err = c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true}) assert.NilError(t, err) From c35376c4558a4d69c872e2a6cba72455f3dcb494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Thu, 17 Aug 2023 16:44:28 +0200 Subject: [PATCH 058/105] volume/local: Don't unmount, restore mounted status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On startup all local volumes were unmounted as a cleanup mechanism for the non-clean exit of the last engine process. This caused live-restored volumes that used special volume opt mount flags to be broken. While the refcount was restored, the _data directory was just unmounted, so all new containers mounting this volume would just have the access to the empty _data directory instead of the real volume. With this patch, the mountpoint isn't unmounted. Instead, if the volume is already mounted, just mark it as mounted, so the next time Mount is called only the ref count is incremented, but no second attempt to mount it is performed. Signed-off-by: Paweł Gronowski (cherry picked from commit 26894844026d20938ec84ea22448172d3af9b652) Signed-off-by: Sebastiaan van Stijn --- volume/local/local.go | 17 +++++++++++++---- volume/local/local_unix.go | 27 +++++++++++++++++++++++---- volume/local/local_windows.go | 5 +++++ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/volume/local/local.go b/volume/local/local.go index 1417941cca3a7..b4f3a3669a842 100644 --- a/volume/local/local.go +++ b/volume/local/local.go @@ -83,13 +83,18 @@ func New(scope string, rootIdentity idtools.Identity) (*Root, error) { quotaCtl: r.quotaCtl, } - // unmount anything that may still be mounted (for example, from an - // unclean shutdown). This is a no-op on windows - unmount(v.path) - if err := v.loadOpts(); err != nil { return nil, err } + + if err := v.restoreIfMounted(); err != nil { + log.G(context.TODO()).WithFields(log.Fields{ + "volume": v.name, + "path": v.path, + "error": err, + }).Warn("restoreIfMounted failed") + } + r.volumes[name] = v } @@ -338,6 +343,10 @@ func (v *localVolume) Unmount(id string) error { return nil } + if !v.active.mounted { + return nil + } + logger.Debug("Unmounting volume") return v.unmount() } diff --git a/volume/local/local_unix.go b/volume/local/local_unix.go index 2db5f8ef09205..0b534653b99c0 100644 --- a/volume/local/local_unix.go +++ b/volume/local/local_unix.go @@ -99,10 +99,6 @@ func (v *localVolume) setOpts(opts map[string]string) error { return v.saveOpts() } -func unmount(path string) { - _ = mount.Unmount(path) -} - func (v *localVolume) needsMount() bool { if v.opts == nil { return false @@ -163,6 +159,29 @@ func (v *localVolume) unmount() error { return nil } +// restoreIfMounted restores the mounted status if the _data directory is already mounted. +func (v *localVolume) restoreIfMounted() error { + if v.needsMount() { + // Check if the _data directory is already mounted. + mounted, err := mountinfo.Mounted(v.path) + if err != nil { + return fmt.Errorf("failed to determine if volume _data path is already mounted: %w", err) + } + + if mounted { + // Mark volume as mounted, but don't increment active count. If + // any container needs this, the refcount will be incremented + // by the live-restore (if enabled). + // In other case, refcount will be zero but the volume will + // already be considered as mounted when Mount is called, and + // only the refcount will be incremented. + v.active.mounted = true + } + } + + return nil +} + func (v *localVolume) CreatedAt() (time.Time, error) { fileInfo, err := os.Stat(v.rootPath) if err != nil { diff --git a/volume/local/local_windows.go b/volume/local/local_windows.go index 43b89b3cb15d5..11723b02f3956 100644 --- a/volume/local/local_windows.go +++ b/volume/local/local_windows.go @@ -43,6 +43,11 @@ func (v *localVolume) postMount() error { return nil } +// restoreIfMounted is a no-op on Windows (because mounts are not supported). +func (v *localVolume) restoreIfMounted() error { + return nil +} + func (v *localVolume) CreatedAt() (time.Time, error) { fileInfo, err := os.Stat(v.rootPath) if err != nil { From 8216da20afc8fd056a179e158174d70fc37f57fd Mon Sep 17 00:00:00 2001 From: Jean-Michel Rouet Date: Fri, 4 Nov 2022 11:27:25 +0100 Subject: [PATCH 059/105] more robust dockerd-rootless-setuptools.sh Fixing case where username may contain a backslash. This case can happen for winbind/samba active directory domain users. Signed-off-by: Jean-Michel Rouet Use more meaningful variable name Signed-off-by: Jean-Michel Rouet Update contrib/dockerd-rootless-setuptool.sh Co-authored-by: Akihiro Suda Signed-off-by: Jean-Michel Rouet Use more meaningful variable name Signed-off-by: Jean-Michel Rouet Update contrib/dockerd-rootless-setuptool.sh Co-authored-by: Akihiro Suda Signed-off-by: Jean-Michel Rouet (cherry picked from commit 2f0ba0a7e51756c9475d8b2379f32e4074e39afc) Signed-off-by: Ameya Gawde --- contrib/dockerd-rootless-setuptool.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/contrib/dockerd-rootless-setuptool.sh b/contrib/dockerd-rootless-setuptool.sh index c82bbc84604af..0c517a810f655 100755 --- a/contrib/dockerd-rootless-setuptool.sh +++ b/contrib/dockerd-rootless-setuptool.sh @@ -37,6 +37,8 @@ BIN="" SYSTEMD="" CFG_DIR="" XDG_RUNTIME_DIR_CREATED="" +USERNAME="" +USERNAME_ESCAPED="" # run checks and also initialize global vars init() { @@ -78,6 +80,11 @@ init() { exit 1 fi + # Set USERNAME from `id -un` and potentially protect backslash + # for windbind/samba domain users + USERNAME=$(id -un) + USERNAME_ESCAPED=$(echo $USERNAME | sed 's/\\/\\\\/g') + # set CFG_DIR CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}" @@ -222,21 +229,21 @@ init() { fi # instructions: validate subuid/subgid files for current user - if ! grep -q "^$(id -un):\|^$(id -u):" /etc/subuid 2> /dev/null; then + if ! grep -q "^$USERNAME_ESCAPED:\|^$(id -u):" /etc/subuid 2> /dev/null; then instructions=$( cat <<- EOI ${instructions} - # Add subuid entry for $(id -un) - echo "$(id -un):100000:65536" >> /etc/subuid + # Add subuid entry for ${USERNAME} + echo "${USERNAME}:100000:65536" >> /etc/subuid EOI ) fi - if ! grep -q "^$(id -un):\|^$(id -u):" /etc/subgid 2> /dev/null; then + if ! grep -q "^$USERNAME_ESCAPED:\|^$(id -u):" /etc/subgid 2> /dev/null; then instructions=$( cat <<- EOI ${instructions} - # Add subgid entry for $(id -un) - echo "$(id -un):100000:65536" >> /etc/subgid + # Add subgid entry for ${USERNAME} + echo "${USERNAME}:100000:65536" >> /etc/subgid EOI ) fi @@ -340,7 +347,7 @@ install_systemd() { ) INFO "Installed ${SYSTEMD_UNIT} successfully." INFO "To control ${SYSTEMD_UNIT}, run: \`systemctl --user (start|stop|restart) ${SYSTEMD_UNIT}\`" - INFO "To run ${SYSTEMD_UNIT} on system startup, run: \`sudo loginctl enable-linger $(id -un)\`" + INFO "To run ${SYSTEMD_UNIT} on system startup, run: \`sudo loginctl enable-linger ${USERNAME}\`" echo } From e2ab5f72eb77212084afc2aa930ccf1ac93d7290 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 8 Sep 2023 16:53:06 +0000 Subject: [PATCH 060/105] 24.0: Update buildkit to fix source policy order This brings in moby/buildkit#4215 which fixes a major issue with source policies in buildkit. Signed-off-by: Brian Goff --- builder/builder-next/worker/worker.go | 2 +- vendor.mod | 2 +- vendor.sum | 4 ++-- .../moby/buildkit/solver/llbsolver/solver.go | 16 +++++----------- vendor/modules.txt | 2 +- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index d91573c93f140..dad7331aeb198 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -50,7 +50,7 @@ import ( ) func init() { - version.Version = "v0.11.6+616c3f613b54" + version.Version = "v0.11.7+d3e6c1360f6e" } const labelCreatedAt = "buildkit/createdat" diff --git a/vendor.mod b/vendor.mod index 7cda78ef2bd90..38ef1bceb5c69 100644 --- a/vendor.mod +++ b/vendor.mod @@ -56,7 +56,7 @@ require ( github.com/klauspost/compress v1.16.3 github.com/miekg/dns v1.1.43 github.com/mistifyio/go-zfs/v3 v3.0.1 - github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go + github.com/moby/buildkit v0.11.7-0.20230908085316-d3e6c1360f6e // IMPORTANT: when updating, also update the version in builder/builder-next/worker/worker.go github.com/moby/ipvs v1.1.0 github.com/moby/locker v1.0.1 github.com/moby/patternmatcher v0.5.0 diff --git a/vendor.sum b/vendor.sum index 04af7872226f0..af7288f89a786 100644 --- a/vendor.sum +++ b/vendor.sum @@ -1043,8 +1043,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ= -github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 h1:LSh03Csyx/zQq8MreC9MYMQE/+5EkohwZMvXSS6kMZo= -github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54/go.mod h1:bMQDryngJKGvJ/ZuRFhrejurbvYSv3NkGCheQ59X4AM= +github.com/moby/buildkit v0.11.7-0.20230908085316-d3e6c1360f6e h1:iDGoHMw0bMy+AVD59fVDM+jH6rivbFKCVF4iZ+Uw1Rc= +github.com/moby/buildkit v0.11.7-0.20230908085316-d3e6c1360f6e/go.mod h1:bMQDryngJKGvJ/ZuRFhrejurbvYSv3NkGCheQ59X4AM= github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go index d65a9e6490c7c..94d25ce5b7b28 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go @@ -977,27 +977,21 @@ func loadEntitlements(b solver.Builder) (entitlements.Set, error) { } func loadSourcePolicy(b solver.Builder) (*spb.Policy, error) { - set := make(map[spb.Rule]struct{}, 0) + var srcPol spb.Policy err := b.EachValue(context.TODO(), keySourcePolicy, func(v interface{}) error { x, ok := v.(spb.Policy) if !ok { return errors.Errorf("invalid source policy %T", v) } for _, f := range x.Rules { - set[*f] = struct{}{} + r := *f + srcPol.Rules = append(srcPol.Rules, &r) } + srcPol.Version = x.Version return nil }) if err != nil { return nil, err } - var srcPol *spb.Policy - if len(set) > 0 { - srcPol = &spb.Policy{} - for k := range set { - k := k - srcPol.Rules = append(srcPol.Rules, &k) - } - } - return srcPol, nil + return &srcPol, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index c3df1de562494..6de240d23534c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -577,7 +577,7 @@ github.com/mistifyio/go-zfs/v3 # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 -# github.com/moby/buildkit v0.11.7-0.20230723230859-616c3f613b54 +# github.com/moby/buildkit v0.11.7-0.20230908085316-d3e6c1360f6e ## explicit; go 1.18 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types From f014c349a0fd00df3bdca57aed124a93d2f8d2de Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 11 Sep 2023 15:47:00 +0200 Subject: [PATCH 061/105] update to go1.20.8 go1.20.8 (released 2023-09-06) includes two security fixes to the html/template package, as well as bug fixes to the compiler, the go command, the runtime, and the crypto/tls, go/types, net/http, and path/filepath packages. See the Go 1.20.8 milestone on our issue tracker for details: https://github.com/golang/go/issues?q=milestone%3AGo1.20.8+label%3ACherryPickApproved full diff: https://github.com/golang/go/compare/go1.20.7...go1.20.8 From the security mailing: [security] Go 1.21.1 and Go 1.20.8 are released Hello gophers, We have just released Go versions 1.21.1 and 1.20.8, minor point releases. These minor releases include 4 security fixes following the security policy: - cmd/go: go.mod toolchain directive allows arbitrary execution The go.mod toolchain directive, introduced in Go 1.21, could be leveraged to execute scripts and binaries relative to the root of the module when the "go" command was executed within the module. This applies to modules downloaded using the "go" command from the module proxy, as well as modules downloaded directly using VCS software. Thanks to Juho Nurminen of Mattermost for reporting this issue. This is CVE-2023-39320 and Go issue https://go.dev/issue/62198. - html/template: improper handling of HTML-like comments within script contexts The html/template package did not properly handle HMTL-like "" comment tokens, nor hashbang "#!" comment tokens, in