Merge branch 'containerd:main' into Adding-regression-tests-for-sandb…

…ox-pods-when-no-CNI-plugins-are-initialized
containerd · Nov 21, 2024 · f2adeb2 · f2adeb2
2 parents 4cd3211 + f0ebbd3
commit f2adeb2
Show file tree

Hide file tree

Showing 49 changed files with 784 additions and 184 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -587,8 +587,7 @@ jobs:
           echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
           sudo sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
           sudo apt-get update
-          # Pinned to 2.4.1-1 until https://github.com/hashicorp/vagrant/pull/13532 in released version
-          sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant=2.4.1-1 ovmf
+          sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant ovmf
           # https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1725#issuecomment-1454058646
           sudo cp /usr/share/OVMF/OVMF_VARS_4M.fd /var/lib/libvirt/qemu/nvram/
           sudo systemctl enable --now libvirtd

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -36,7 +36,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+        uses: github/codeql-action/init@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
         # Override language selection by uncommenting this and choosing your languages
         # with:
         #   languages: go, javascript, csharp, python, cpp, java
@@ -46,4 +46,4 @@ jobs:
           make
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+        uses: github/codeql-action/analyze@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # v3.27.4
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
@@ -49,6 +49,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@4f3212b61783c3c68e8309a0f18a699764811cda # tag=v3.27.1
+        uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # tag=v3.27.4
         with:
           sarif_file: results.sarif
diff --git a/Makefile b/Makefile
@@ -248,6 +248,11 @@ bin/runc-fp: integration/failpoint/cmd/runc-fp FORCE
 	@echo "$(WHALE) $@"
 	@$(GO) build ${GO_BUILD_FLAGS} -o $@ ./integration/failpoint/cmd/runc-fp
 
+# build loopback-v2 with failpoint support, only used by integration test
+bin/loopback-v2: integration/failpoint/cmd/loopback-v2 FORCE
+	@echo "$(WHALE) $@"
+	@CGO_ENABLED=${SHIM_CGO_ENABLED} $(GO) build ${GO_BUILD_FLAGS} -o $@ ./integration/failpoint/cmd/loopback-v2
+
 benchmark: ## run benchmarks tests
 	@echo "$(WHALE) $@"
 	@$(GO) test ${TESTFLAGS} -bench . -run Benchmark -test.root

diff --git a/RELEASES.md b/RELEASES.md
@@ -443,13 +443,13 @@ The deprecated features are shown in the following table:
 | Built-in `aufs` snapshotter                                                      | containerd v1.5     | containerd v2.0 ✅                    | Use `overlayfs` snapshotter              |
 | Container label `containerd.io/restart.logpath`                                  | containerd v1.5     | containerd v2.0 ✅                    | Use `containerd.io/restart.loguri` label |
 | `cri-containerd-*.tar.gz` release bundles                                        | containerd v1.6     | containerd v2.0 ✅                    | Use `containerd-*.tar.gz` bundles        |
-| Pulling Schema 1 images (`application/vnd.docker.distribution.manifest.v1+json`) | containerd v1.7     | containerd v2.1 (Disabled in v2.0 ✅) | Use Schema 2 or OCI images               |
+| Pulling Schema 1 images (`application/vnd.docker.distribution.manifest.v1+prettyjws`) | containerd v1.7     | containerd v2.1 (Disabled in v2.0 ✅) | Use Schema 2 or OCI images               |
 | CRI `v1alpha2`                                                                   | containerd v1.7     | containerd v2.0 ✅                    | Use CRI `v1`                             |
 | Legacy CRI implementation of podsandbox support                                  | containerd v2.0     | containerd v2.0 ✅                    |                                          |
 | Go-Plugin library (`*.so`) as containerd runtime plugin                          | containerd v2.0     | containerd v2.1                       | Use external plugins (proxy or binary)   |
 
 - Pulling Schema 1 images has been disabled in containerd v2.0, but it still can be enabled by setting an environment variable `CONTAINERD_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE=1`
-  until containerd v2.1. `ctr` users have to specify `--local` too (e.g., `ctr images pull --local`).
+  until containerd v2.1. `ctr` users have to specify `--local` too (e.g., `ctr images pull --local`). Users of CRI clients (such as Kubernetes and `crictl`) have to specify this environment variable on the containerd daemon (usually in the systemd unit).
 
 ### Deprecated config properties
 The deprecated properties in [`config.toml`](./docs/cri/config.md) are shown in the following table:

diff --git a/cmd/containerd-stress/main.go b/cmd/containerd-stress/main.go
@@ -29,6 +29,7 @@ import (
 	"time"
 
 	containerd "github.com/containerd/containerd/v2/client"
+	"github.com/containerd/containerd/v2/defaults"
 	"github.com/containerd/containerd/v2/integration/remote"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	"github.com/containerd/containerd/v2/plugins"
@@ -137,7 +138,7 @@ func main() {
 		&cli.StringFlag{
 			Name:    "address",
 			Aliases: []string{"a"},
-			Value:   "/run/containerd/containerd.sock",
+			Value:   defaults.DefaultAddress,
 			Usage:   "Path to the containerd socket",
 		},
 		&cli.IntFlag{

diff --git a/contrib/v2-migrate.sh b/contrib/v2-migrate.sh
@@ -65,14 +65,15 @@ for GOFILE in $(find . -name "*.go" | grep -v "./vendor/" ); do
   perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/metrics/$1$2"github.com\/containerd\/containerd\/v2\/core\/metrics/g' $GOFILE
   perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/runtime/$1$2"github.com\/containerd\/containerd\/v2\/core\/runtime/g' $GOFILE
   perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/core\/runtime\/v2\/runc\/options/$1$2"github.com\/containerd\/containerd\/api\/types\/runc\/options/g' $GOFILE
-  perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/pkg\/runtimeoptions/$1$2"github.com\/containerd\/containerd\/v2\/types\/runtimeoptions/g' $GOFILE
+  perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/pkg\/runtimeoptions/$1$2"github.com\/containerd\/containerd\/api\/types\/runtimeoptions/g' $GOFILE
   perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/protobuf/$1$2"github.com\/containerd\/containerd\/v2\/pkg\/protobuf/g' $GOFILE
   perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/api/$1$2"github.com\/containerd\/containerd\/api/g' $GOFILE
 
   # Migrate packages split out to their own repository
   perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/platforms/$1$2"github.com\/containerd\/platforms/g' $GOFILE
   perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/pkg\/errdefs/$1$2"github.com\/containerd\/errdefs/g' $GOFILE
   perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/plugin(\/|")/$1$2"github.com\/containerd\/plugin$3/g' $GOFILE
+  perl -pi -e 's/([\t]|[ ]{2,8}|import )([_a-zA-Z0-9]+ )?"github\.com\/containerd\/containerd\/v2\/pkg\/userns/$1$2"github.com\/moby\/sys\/user\/userns/g' $GOFILE
 
   gofmt -s -w $GOFILE
 done
diff --git a/core/metadata/content.go b/core/metadata/content.go
@@ -25,6 +25,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	eventstypes "github.com/containerd/containerd/api/events"
 	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	digest "github.com/opencontainers/go-digest"
@@ -209,7 +210,7 @@ func (cs *contentStore) Delete(ctx context.Context, dgst digest.Digest) error {
 	cs.l.RLock()
 	defer cs.l.RUnlock()
 
-	return update(ctx, cs.db, func(tx *bolt.Tx) error {
+	if err := update(ctx, cs.db, func(tx *bolt.Tx) error {
 		bkt := getBlobBucket(tx, ns, dgst)
 		if bkt == nil {
 			return fmt.Errorf("content digest %v: %w", dgst, errdefs.ErrNotFound)
@@ -227,7 +228,18 @@ func (cs *contentStore) Delete(ctx context.Context, dgst digest.Digest) error {
 		cs.db.dirtyCS = true
 
 		return nil
-	})
+	}); err != nil {
+		return err
+	}
+
+	if publisher := cs.db.Publisher(ctx); publisher != nil {
+		if err := publisher.Publish(ctx, "/content/delete", &eventstypes.ContentDelete{
+			Digest: dgst.String(),
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func (cs *contentStore) ListStatuses(ctx context.Context, fs ...string) ([]content.Status, error) {

diff --git a/defaults/defaults_darwin.go b/defaults/defaults_darwin.go
@@ -17,21 +17,6 @@
 package defaults
 
 const (
-	// DefaultRootDir is the default location used by containerd to store
-	// persistent data
-	DefaultRootDir = "/var/lib/containerd"
-	// DefaultStateDir is the default location used by containerd to store
-	// transient data
-	DefaultStateDir = "/var/run/containerd"
-	// DefaultAddress is the default unix socket address
-	DefaultAddress = "/var/run/containerd/containerd.sock"
-	// DefaultDebugAddress is the default unix socket address for pprof data
-	DefaultDebugAddress = "/var/run/containerd/debug.sock"
-	// DefaultFIFODir is the default location used by client-side cio library
-	// to store FIFOs.
-	DefaultFIFODir = "/var/run/containerd/fifo"
 	// DefaultRuntime would be a multiple of choices, thus empty
 	DefaultRuntime = ""
-	// DefaultConfigDir is the default location for config files.
-	DefaultConfigDir = "/etc/containerd"
 )
diff --git a/defaults/defaults_linux.go b/defaults/defaults_linux.go
@@ -17,6 +17,20 @@
 package defaults
 
 const (
+	// DefaultAddress is the default unix socket address
+	DefaultAddress = "/run/containerd/containerd.sock"
+	// DefaultDebugAddress is the default unix socket address for pprof data
+	DefaultDebugAddress = "/run/containerd/debug.sock"
+	// DefaultFIFODir is the default location used by client-side cio library
+	// to store FIFOs.
+	DefaultFIFODir = "/run/containerd/fifo"
 	// DefaultRuntime is the default linux runtime
 	DefaultRuntime = "io.containerd.runc.v2"
+	// DefaultSnapshotter will set the default snapshotter for the platform.
+	// This will be based on the client compilation target, so take that into
+	// account when choosing this value.
+	DefaultSnapshotter = "overlayfs"
+	// DefaultStateDir is the default location used by containerd to store
+	// transient data
+	DefaultStateDir = "/run/containerd"
 )
diff --git a/defaults/defaults_snapshotter_windows.go b/defaults/defaults_snapshotter_windows.go
diff --git a/defaults/defaults_unix.go b/defaults/defaults_unix.go
@@ -1,4 +1,4 @@
-//go:build !windows && !darwin
+//go:build unix
 
 /*
    Copyright The containerd Authors.
@@ -19,19 +19,9 @@
 package defaults
 
 const (
+	// DefaultConfigDir is the default location for config files.
+	DefaultConfigDir = "/etc/containerd"
 	// DefaultRootDir is the default location used by containerd to store
 	// persistent data
 	DefaultRootDir = "/var/lib/containerd"
-	// DefaultStateDir is the default location used by containerd to store
-	// transient data
-	DefaultStateDir = "/run/containerd"
-	// DefaultAddress is the default unix socket address
-	DefaultAddress = "/run/containerd/containerd.sock"
-	// DefaultDebugAddress is the default unix socket address for pprof data
-	DefaultDebugAddress = "/run/containerd/debug.sock"
-	// DefaultFIFODir is the default location used by client-side cio library
-	// to store FIFOs.
-	DefaultFIFODir = "/run/containerd/fifo"
-	// DefaultConfigDir is the default location for config files.
-	DefaultConfigDir = "/etc/containerd"
 )
diff --git a/defaults/defaults_snapshotter_unix.go → defaults/defaults_unix_nolinux.go b/defaults/defaults_snapshotter_unix.go → defaults/defaults_unix_nolinux.go
@@ -1,4 +1,4 @@
-//go:build darwin || freebsd || solaris
+//go:build unix && !linux
 
 /*
    Copyright The containerd Authors.
@@ -19,8 +19,18 @@
 package defaults
 
 const (
+	// DefaultAddress is the default unix socket address
+	DefaultAddress = "/var/run/containerd/containerd.sock"
+	// DefaultDebugAddress is the default unix socket address for pprof data
+	DefaultDebugAddress = "/var/run/containerd/debug.sock"
+	// DefaultFIFODir is the default location used by client-side cio library
+	// to store FIFOs.
+	DefaultFIFODir = "/var/run/containerd/fifo"
 	// DefaultSnapshotter will set the default snapshotter for the platform.
 	// This will be based on the client compilation target, so take that into
 	// account when choosing this value.
 	DefaultSnapshotter = "native"
+	// DefaultStateDir is the default location used by containerd to store
+	// transient data
+	DefaultStateDir = "/var/run/containerd"
 )
diff --git a/defaults/defaults_windows.go b/defaults/defaults_windows.go
@@ -38,9 +38,16 @@ const (
 	DefaultAddress = `\\.\pipe\containerd-containerd`
 	// DefaultDebugAddress is the default winpipe address for pprof data
 	DefaultDebugAddress = `\\.\pipe\containerd-debug`
+	// DefaultDiffer will set the default differ for the platform.
+	// This differ should be compatible with the windows snapshotter.
+	DefaultDiffer = "windows"
 	// DefaultFIFODir is the default location used by client-side cio library
 	// to store FIFOs. Unused on Windows.
 	DefaultFIFODir = ""
 	// DefaultRuntime is the default windows runtime
 	DefaultRuntime = "io.containerd.runhcs.v1"
+	// DefaultSnapshotter will set the default snapshotter for the platform.
+	// This will be based on the client compilation target, so take that into
+	// account when choosing this value.
+	DefaultSnapshotter = "windows"
 )
diff --git a/docs/containerd-2.0.md b/docs/containerd-2.0.md
@@ -99,7 +99,7 @@ Administrators whose workloads are running on containerd versions >= 1.6.27, >=
 
 ### Docker Schema 1 image support is disabled by default
 
-Pulling Docker Schema 1 (`application/vnd.docker.distribution.manifest.v1+json` or `application/vnd.docker.distribution.manifest.v1+prettyjws`) images is disabled by default. Users should migrate their container images by rebuilding/pushing with the latest Docker or nerdctl+Buildkit tooling. Previous behavior can be re-enabled by setting an environment variable `CONTAINERD_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE=1` for `containerd` (in the case of CRI) and `ctr`; however, users are **strongly recommended** to migrate to Docker Schema 2 or OCI images. Support for Docker Schema 1 images will be fully removed in a future release.
+Pulling Docker Schema 1 (`application/vnd.docker.distribution.manifest.v1+prettyjws`) images is disabled by default. Users should migrate their container images by rebuilding/pushing with the latest Docker or nerdctl+Buildkit tooling. Previous behavior can be re-enabled by setting an environment variable `CONTAINERD_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE=1` for `containerd` (in the case of CRI clients, such as Kubernetes and `crictl`) and `ctr` (`ctr` users also must specify `--local`); however, users are **strongly recommended** to migrate to Docker Schema 2 or OCI images. Support for Docker Schema 1 images will be fully removed in a future release.
 
 Since containerd 1.7.8 and 1.6.25, schema 1 images are labeled during pull with `io.containerd.image/converted-docker-schema1`. To find images that were converted from schema 1, you can use a command like `ctr namespaces list --quiet | xargs -I{} -- ctr --namespace={} image list 'labels."io.containerd.image/converted-docker-schema1"'`.
 

diff --git a/go.mod b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/containerd/cgroups/v3 v3.0.3
 	github.com/containerd/console v1.0.4
 	github.com/containerd/containerd/api v1.8.0
-	github.com/containerd/continuity v0.4.4
+	github.com/containerd/continuity v0.4.5
 	github.com/containerd/errdefs v1.0.0
 	github.com/containerd/errdefs/pkg v0.3.0
 	github.com/containerd/fifo v1.1.0
@@ -77,7 +77,7 @@ require (
 	golang.org/x/sys v0.27.0
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38
 	google.golang.org/grpc v1.67.1
-	google.golang.org/protobuf v1.35.1
+	google.golang.org/protobuf v1.35.2
 	k8s.io/apimachinery v0.31.2
 	k8s.io/client-go v0.31.2
 	k8s.io/component-base v0.31.2

diff --git a/go.sum b/go.sum
@@ -671,8 +671,8 @@ github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn
 github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0=
 github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc=
-github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII=
-github.com/containerd/continuity v0.4.4/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
+github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4=
+github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
 github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
 github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
@@ -1832,8 +1832,8 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
-google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

diff --git a/defaults/defaults_snapshotter_linux.go → ...gration/failpoint/cmd/loopback-v2/main.go b/defaults/defaults_snapshotter_linux.go → ...gration/failpoint/cmd/loopback-v2/main.go
@@ -14,11 +14,29 @@
    limitations under the License.
 */
 
-package defaults
+package main
 
-const (
-	// DefaultSnapshotter will set the default snapshotter for the platform.
-	// This will be based on the client compilation target, so take that into
-	// account when choosing this value.
-	DefaultSnapshotter = "overlayfs"
+import (
+	"fmt"
+	"log"
+	"net"
+
+	"github.com/vishvananda/netlink"
 )
+
+// isLoInterfaceUp validates whether the lo interface is up
+func isLoInterfaceUp() (bool, error) {
+	link, err := netlink.LinkByName("lo")
+	if err != nil {
+		return false, fmt.Errorf("could not find interface lo: %w", err)
+	}
+	return link.Attrs().Flags&net.FlagUp != 0, nil
+}
+
+func main() {
+	up, err := isLoInterfaceUp()
+	if err != nil {
+		log.Fatalf("could not check lo interface status: %v", err)
+	}
+	fmt.Printf("Loopback interface is %s\n", map[bool]string{true: "UP", false: "DOWN"}[up])
+}