Skip to content

Commit

Permalink
refactor node image build and fix pause image
Browse files Browse the repository at this point in the history
  • Loading branch information
@BenTheElder
BenTheElder committed Apr 23, 2020
1 parent b10d2c7 commit 588de78
Show file tree
Hide file tree
Showing 29 changed files with 272 additions and 159 deletions.
13 changes: 0 additions & 13 deletions images/base/files/etc/containerd/config.toml
View file

This file was deleted.

10 changes: 5 additions & 5 deletions pkg/build/build.go → pkg/build/nodeimage/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,22 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package build
package nodeimage

import (
"runtime"

"sigs.k8s.io/kind/pkg/build/internal/kube"
"sigs.k8s.io/kind/pkg/build/nodeimage/internal/kube"
"sigs.k8s.io/kind/pkg/errors"
"sigs.k8s.io/kind/pkg/log"
)

// NodeImage builds a node image using the supplied options
func NodeImage(options ...Option) error {
// Build builds a node image using the supplied options
func Build(options ...Option) error {
// default options
ctx := &buildContext{
mode: DefaultMode,
image: DefaultNodeImage,
image: DefaultImage,
baseImage: DefaultBaseImage,
logger: log.NoopLogger{},
// TODO: only host arch supported. changing this will be tricky
Expand Down
156 changes: 37 additions & 119 deletions pkg/build/build_impl.go → pkg/build/nodeimage/build_impl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package build
package nodeimage

import (
"fmt"
Expand All @@ -28,8 +28,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"

"sigs.k8s.io/kind/pkg/build/internal/container/docker"
"sigs.k8s.io/kind/pkg/build/internal/kube"
"sigs.k8s.io/kind/pkg/build/nodeimage/internal/container/docker"
"sigs.k8s.io/kind/pkg/errors"
"sigs.k8s.io/kind/pkg/exec"
"sigs.k8s.io/kind/pkg/fs"
Expand Down Expand Up @@ -97,41 +96,6 @@ func (c *buildContext) getBuiltImages() (sets.String, error) {
return images, nil
}

// private kube.InstallContext implementation, local to the image build
type installContext struct {
basePath string
containerID string
}

var _ kube.InstallContext = &installContext{}

func (ic *installContext) BasePath() string {
return ic.basePath
}

func (ic *installContext) Run(command string, args ...string) error {
cmd := exec.Command(
"docker",
append(
[]string{"exec", ic.containerID, command},
args...,
)...,
)
exec.InheritOutput(cmd)
return cmd.Run()
}

func (ic *installContext) CombinedOutputLines(command string, args ...string) ([]string, error) {
cmd := exec.Command(
"docker",
append(
[]string{"exec", ic.containerID, command},
args...,
)...,
)
return exec.CombinedOutputLines(cmd)
}

func (c *buildContext) buildImage(dir string) error {
// build the image, tagged as tagImageAs, using the our tempdir as the context
c.logger.V(0).Info("Starting image build ...")
Expand Down Expand Up @@ -197,7 +161,7 @@ func (c *buildContext) buildImage(dir string) error {
}

// setup the kubelet dropin
kubeletDropin := "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
const kubeletDropin = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
if err := createFile(cmder, kubeletDropin, kubeadm10conf); err != nil {
return errors.Wrap(err, "failed to configure kubelet service")
}
Expand All @@ -211,11 +175,28 @@ func (c *buildContext) buildImage(dir string) error {
}

// pre-pull images that were not part of the build
if err = c.prePullImages(dir, containerID); err != nil {
images, err := c.prePullImages(dir, containerID)
if err != nil {
c.logger.Errorf("Image build Failed! Failed to pull Images: %v", err)
return err
}

// find the pause image and inject containerd config
pauseImage := findSandboxImage(images)
if pauseImage == "" {
return errors.New("failed to find imported pause image")
}
containerdConfig, err := getContainerdConfig(containerdConfigTemplateData{
SandboxImage: pauseImage,
})
if err != nil {
return err
}
const containerdConfigPath = "/etc/containerd/config.toml"
if err := createFile(cmder, containerdConfigPath, containerdConfig); err != nil {
return err
}

// Save the image changes to a new image
cmd := exec.Command(
"docker", "commit",
Expand All @@ -233,29 +214,13 @@ func (c *buildContext) buildImage(dir string) error {
return nil
}

func createFile(containerCmder exec.Cmder, filePath, contents string) error {
// ensure the directory first
// NOTE: the paths inside the container should use the path package
// and not filepath (!), we want posixy paths in the linux container, NOT
// whatever path format the host uses. For paths on the host we use filepath
if err := containerCmder.Command("mkdir", "-p", path.Dir(filePath)).Run(); err != nil {
return err
}

return containerCmder.Command(
"cp", "/dev/stdin", filePath,
).SetStdin(
strings.NewReader(contents),
).Run()
}

// must be run after kubernetes has been installed on the node
func (c *buildContext) prePullImages(dir, containerID string) error {
func (c *buildContext) prePullImages(dir, containerID string) ([]string, error) {
// first get the images we actually built
builtImages, err := c.getBuiltImages()
if err != nil {
c.logger.Errorf("Image build Failed! Failed to get built images: %v", err)
return err
return nil, err
}

// helpers to run things in the build container
Expand All @@ -266,11 +231,11 @@ func (c *buildContext) prePullImages(dir, containerID string) error {
rawVersion, err := exec.CombinedOutputLines(cmder.Command("cat", kubernetesVersionLocation))
if err != nil {
c.logger.Errorf("Image build Failed! Failed to get Kubernetes version: %v", err)
return err
return nil, err
}
if len(rawVersion) != 1 {
c.logger.Errorf("Image build Failed! Failed to get Kubernetes version: %v", err)
return errors.New("invalid kubernetes version file")
return nil, errors.New("invalid kubernetes version file")
}

// before Kubernetes v1.12.0 kubeadm requires arch specific images, instead
Expand All @@ -279,7 +244,7 @@ func (c *buildContext) prePullImages(dir, containerID string) error {
// so we virtually re-tag them here.
ver, err := version.ParseSemantic(rawVersion[0])
if err != nil {
return err
return nil, err
}

// get image tag fixing function for this version
Expand All @@ -291,7 +256,7 @@ func (c *buildContext) prePullImages(dir, containerID string) error {
for _, image := range builtImages.List() {
registry, tag, err := docker.SplitImage(image)
if err != nil {
return err
return nil, err
}
registry = fixRepository(registry)
fixedImages.Insert(registry + ":" + tag)
Expand All @@ -304,21 +269,21 @@ func (c *buildContext) prePullImages(dir, containerID string) error {
"kubeadm", "config", "images", "list", "--kubernetes-version", rawVersion[0],
))
if err != nil {
return err
return nil, err
}

// write the default CNI manifest
if err := writeManifest(cmder, defaultCNIManifestLocation, defaultCNIManifest); err != nil {
if err := createFile(cmder, defaultCNIManifestLocation, defaultCNIManifest); err != nil {
c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err)
return err
return nil, err
}
// all builds should install the default CNI images from the above manifest currently
requiredImages = append(requiredImages, defaultCNIImages...)

// write the default Storage manifest
if err := writeManifest(cmder, defaultStorageManifestLocation, defaultStorageManifest); err != nil {
if err := createFile(cmder, defaultStorageManifestLocation, defaultStorageManifest); err != nil {
c.logger.Errorf("Image build Failed! Failed write default Storage Manifest: %v", err)
return err
return nil, err
}
// all builds should install the default storage driver images currently
requiredImages = append(requiredImages, defaultStorageImages...)
Expand All @@ -327,7 +292,7 @@ func (c *buildContext) prePullImages(dir, containerID string) error {
imagesDir := path.Join(dir, "bits", "images")
if err := os.MkdirAll(imagesDir, 0777); err != nil {
c.logger.Errorf("Image build Failed! Failed create local images dir: %v", err)
return errors.Wrap(err, "failed to make images dir")
return nil, errors.Wrap(err, "failed to make images dir")
}

fns := []func() error{}
Expand All @@ -354,7 +319,7 @@ func (c *buildContext) prePullImages(dir, containerID string) error {
})
}
if err := errors.AggregateConcurrent(fns); err != nil {
return err
return nil, err
}
close(pulledImages)
pulled := []string{}
Expand All @@ -366,7 +331,7 @@ func (c *buildContext) prePullImages(dir, containerID string) error {
importer := newContainerdImporter(cmder)
if err := importer.Prepare(); err != nil {
c.logger.Errorf("Image build Failed! Failed to prepare containerd to load images %v", err)
return err
return nil, err
}

// TODO: return this error?
Expand Down Expand Up @@ -411,57 +376,10 @@ func (c *buildContext) prePullImages(dir, containerID string) error {
// run all image loading concurrently until one fails or all succeed
if err := errors.UntilErrorConcurrent(loadFns); err != nil {
c.logger.Errorf("Image build Failed! Failed to load images %v", err)
return err
return nil, err
}

return nil
}

func writeManifest(cmder exec.Cmder, manifestPath, manifestContents string) error {
// NOTE: the paths inside the container should use the path package
// and not filepath (!), we want posixy paths in the linux container, NOT
// whatever path format the host uses. For paths on the host we use filepath
cmdMkdir := cmder.Command("mkdir", "-p", path.Dir(manifestPath))
exec.InheritOutput(cmdMkdir)
if err := cmdMkdir.Run(); err != nil {
return err
}

return cmder.Command(
"cp", "/dev/stdin", manifestPath,
).SetStdin(
strings.NewReader(manifestContents),
).Run()
}

func repositoryCorrectorForVersion(kubeVersion *version.Version, arch string) func(string) string {
archSuffix := "-" + arch

// For kubernetes v1.15+ (actually 1.16 alpha versions) we may need to
// drop the arch suffix from images to get the expected image
// for < v1.12 we need to do the opposite.
// We can accomplish this by just handling < 1.12 & >= 1.12 as we won't
// touch images that match the expectation in either case ...

if kubeVersion.LessThan(version.MustParseSemantic("v1.12.0")) {
return func(repository string) string {
if !strings.HasSuffix(repository, archSuffix) {
fixed := repository + archSuffix
fmt.Println("fixed: " + repository + " -> " + fixed)
repository = fixed
}
return repository
}
}

return func(repository string) string {
if strings.HasSuffix(repository, archSuffix) {
fixed := strings.TrimSuffix(repository, archSuffix)
fmt.Println("fixed: " + repository + " -> " + fixed)
repository = fixed
}
return repository
}
return importer.ListImported()
}

func (c *buildContext) createBuildContainer(buildDir string) (id string, err error) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/const.go → pkg/build/nodeimage/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package build
package nodeimage

// these are well known paths within the node image
const (
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/const_cni.go → pkg/build/nodeimage/const_cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package build
package nodeimage

/*
The default CNI manifest and images are our own tiny kindnet
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/const_storage.go → pkg/build/nodeimage/const_storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package build
package nodeimage

/*
The default PV driver manifest and images are provisionally rancher.io/local-path-provisioner
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/const_systemd.go → pkg/build/nodeimage/const_systemd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package build
package nodeimage

// slightly modified from
// https://github.com/kubernetes/kubernetes/blob/ba8fcafaf8c502a454acd86b728c857932555315/build/debs/kubelet.service
Expand Down
Loading

0 comments on commit 588de78

Please sign in to comment.