From 45e1ae024e7fc316e4dab1817e18f02df433ad37 Mon Sep 17 00:00:00 2001 From: Steve Hiehn Date: Mon, 20 Aug 2018 17:40:49 -0300 Subject: [PATCH] Run analyze and export directly rather than through docker Signed-off-by: Dave Goddard --- acceptance/acceptance_test.go | 52 +++++++----------------- analyzer.go | 35 ++++++++++++++++ build.go | 75 ++++------------------------------- cmd/pack/main.go | 6 +-- exporter.go | 59 +++++++++++++++++++++++++++ go.mod | 18 +++++++-- go.sum | 34 ++++++++++++++++ utils.go | 55 +++++++++++++++++++++++++ 8 files changed, 222 insertions(+), 112 deletions(-) create mode 100644 analyzer.go create mode 100644 exporter.go create mode 100644 utils.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index a2e2d0f1d5..5f4e30ac4c 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -2,8 +2,6 @@ package acceptance_test import ( "bytes" - "encoding/hex" - "errors" "io/ioutil" "os/exec" "path/filepath" @@ -42,12 +40,6 @@ func TestPack(t *testing.T) { defer os.RemoveAll(packTmpDir) } - hostMachineIP, err := findHostMachineIP() - if err != nil { - panic(err) - } - os.Setenv("PACK_HOST_MACHINE_IP", hostMachineIP) - spec.Run(t, "pack", testPack, spec.Report(report.Terminal{})) } @@ -64,6 +56,12 @@ func testPack(t *testing.T, when spec.G, it spec.S) { if err != nil { t.Fatal(err) } + if err := os.Mkdir(filepath.Join(homeDir, ".docker"), 0777); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(homeDir, ".docker", "config.json"), []byte("{}"), 0666); err != nil { + t.Fatal(err) + } }) it.After(func() { os.RemoveAll(homeDir) @@ -141,19 +139,18 @@ func testPack(t *testing.T, when spec.G, it spec.S) { when("'--publish' flag is specified", func() { it.Before(func() { t.Log("push v3/packs:run to local registry") - for _, name := range []string{"analyze", "build", "export", "run"} { - run(t, exec.Command("docker", "tag", "packs/v3:"+name, fmt.Sprintf("host-machine.local:%s/packs/v3:%s", registryPort, name))) + for _, name := range []string{"build", "run"} { + run(t, exec.Command("docker", "tag", "packs/v3:"+name, fmt.Sprintf("localhost:%s/packs/v3:%s", registryPort, name))) } - run(t, exec.Command("docker", "tag", "packs/v3:run", fmt.Sprintf("localhost:%s/packs/v3:run", registryPort))) run(t, exec.Command("docker", "push", fmt.Sprintf("localhost:%s/packs/v3:run", registryPort))) // Build copy of packs/v3:detect with all group repositories pointing to image in local registry - cmd := exec.Command("docker", "build", "-t", fmt.Sprintf("host-machine.local:%s/packs/v3:detect", registryPort), "-") + cmd := exec.Command("docker", "build", "-t", fmt.Sprintf("localhost:%s/packs/v3:detect", registryPort), "-") cmd.Stdin = bytes.NewReader([]byte(fmt.Sprintf(` FROM packs/v3:detect USER root - RUN sed -i 's/"packs\/v3"/"host-machine.local:%s\/packs\/v3"/' /buildpacks/order.toml + RUN sed -i 's/"packs\/v3"/"localhost:%s\/packs\/v3"/' /buildpacks/order.toml USER packs `, registryPort))) @@ -161,8 +158,8 @@ func testPack(t *testing.T, when spec.G, it spec.S) { }) it.After(func() { - for _, name := range []string{"detect", "analyze", "build", "export", "run"} { - exec.Command("docker", "rmi", fmt.Sprintf("host-machine.local:%s/packs/v3:%s", registryPort, name)).Run() + for _, name := range []string{"detect", "build", "run"} { + exec.Command("docker", "rmi", fmt.Sprintf("localhost:%s/packs/v3:%s", registryPort, name)).Run() } }) @@ -173,14 +170,13 @@ func testPack(t *testing.T, when spec.G, it spec.S) { t.Log("run pack build") cmd := exec.Command( pack, "build", - fmt.Sprintf("host-machine.local:%s/%s", registryPort, repo), + fmt.Sprintf("localhost:%s/%s", registryPort, repo), "-p", sourceCodePath, - "--detect-image", fmt.Sprintf("host-machine.local:%s/packs/v3:detect", registryPort), - "--analyze-image", fmt.Sprintf("host-machine.local:%s/packs/v3:analyze", registryPort), - "--export-image", fmt.Sprintf("host-machine.local:%s/packs/v3:export", registryPort), + "--detect-image", fmt.Sprintf("localhost:%s/packs/v3:detect", registryPort), "--publish", ) cmd.Env = append(os.Environ(), "HOME="+homeDir) + run(t, cmd) t.Log("Checking that registry has contents") @@ -263,21 +259,3 @@ func assertEq(t *testing.T, actual, expected interface{}) { t.Fatal(diff) } } - -func findHostMachineIP() (string, error) { - txt, err := exec.Command("docker", "run", "ubuntu:18.04", "cat", "/proc/net/route").Output() - if err != nil { - return "", err - } - for _, line := range strings.Split(string(txt), "\n") { - arr := strings.Split(line, "\t") - if len(arr) > 2 && arr[1] == "00000000" { - b, err := hex.DecodeString(arr[2]) - if err != nil { - return "", err - } - return fmt.Sprintf("%d.%d.%d.%d", b[3], b[2], b[1], b[0]), nil - } - } - return "", errors.New("Could not determine host machine ip") -} diff --git a/analyzer.go b/analyzer.go new file mode 100644 index 0000000000..33ca600b4d --- /dev/null +++ b/analyzer.go @@ -0,0 +1,35 @@ +package pack + +import ( + "os" + + "github.com/buildpack/lifecycle" + "github.com/buildpack/packs" +) + +func analyzer(group lifecycle.BuildpackGroup, launchDir, repoName string, useDaemon bool) error { + origImage, err := readImage(repoName, useDaemon) + if err != nil { + return err + } + + if origImage == nil { + // no previous image to analyze + return nil + } + + analyzer := &lifecycle.Analyzer{ + Buildpacks: group.Buildpacks, + Out: os.Stdout, + Err: os.Stderr, + } + err = analyzer.Analyze( + launchDir, + origImage, + ) + if err != nil { + return packs.FailErrCode(err, packs.CodeFailedBuild) + } + + return nil +} diff --git a/build.go b/build.go index c2c99c5721..4874bacc20 100644 --- a/build.go +++ b/build.go @@ -1,7 +1,6 @@ package pack import ( - "bytes" "crypto/md5" "fmt" "io" @@ -10,11 +9,9 @@ import ( "os/exec" "path/filepath" "runtime" - - "github.com/BurntSushi/toml" ) -func Build(appDir, detectImage, analyzeImage, exportImage, repoName, hostMachineIP string, publish bool) error { +func Build(appDir, detectImage, repoName string, publish bool) error { tempDir, err := ioutil.TempDir("/tmp", "lifecycle.pack.build.") if err != nil { return err @@ -48,32 +45,13 @@ func Build(appDir, detectImage, analyzeImage, exportImage, repoName, hostMachine return err } - fmt.Println("*** ANALYZING: Reading information from previous image for possible re-use") - // TODO: We assume this will need root to access docker.sock, (if so need to chown afterwards) - args := []string{ - "run", - "-v", "/var/run/docker.sock:/var/run/docker.sock", - "-v", filepath.Join(tempDir, "docker-config.json") + ":/home/packs/.docker/config.json", - "-v", filepath.Join(tempDir, "docker-config.json") + ":/etc/docker/daemon.json", - "-v", filepath.Join(tempDir, "launch") + ":/launch", - "-v", filepath.Join(tempDir, "workspace") + ":/workspace:ro", - analyzeImage, - } - if hostMachineIP != "" { - args = append([]string{args[0], "--add-host", "host-machine.local:" + hostMachineIP}, args[1:]...) - } - if !publish { - args = append(args, "-daemon") - } - args = append(args, repoName) - if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil { - fmt.Println(string(out)) + group, err := groupToml(tempDir, detectImage) + if err != nil { return err } - // Read groupRepoImage from ENV:PACK_BP_GROUP_PATH - groupRepoImage, err := groupTomlRepository(tempDir, detectImage) - if err != nil { + fmt.Println("*** ANALYZING: Reading information from previous image for possible re-use") + if err := analyzer(group, filepath.Join(tempDir, "launch"), repoName, !publish); err != nil { return err } @@ -83,7 +61,7 @@ func Build(appDir, detectImage, analyzeImage, exportImage, repoName, hostMachine "-v", filepath.Join(tempDir, "workspace")+":/workspace", "-v", cacheDir+":/cache", "-v", filepath.Join(tempDir, "platform")+":/platform", - groupRepoImage+":build", + group.Repository+":build", ) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -97,52 +75,13 @@ func Build(appDir, detectImage, analyzeImage, exportImage, repoName, hostMachine } fmt.Println("*** EXPORTING:") - args = []string{ - "run", - "--user", "0", - "-v", "/var/run/docker.sock:/var/run/docker.sock", - "-v", filepath.Join(tempDir, "docker-config.json") + ":/root/.docker/config.json", - "-v", filepath.Join(tempDir, "docker-config.json") + ":/etc/docker/daemon.json", - "-v", filepath.Join(tempDir, "launch") + ":/launch:ro", - "-v", filepath.Join(tempDir, "workspace") + ":/workspace:ro", - exportImage, - "-stack", groupRepoImage, - } - if hostMachineIP != "" { - args = append([]string{args[0], "--add-host", "host-machine.local:" + hostMachineIP}, args[1:]...) - } - if !publish { - // TODO: We probably don't want daemon-stack by default - args = append(args, "-daemon", "-daemon-stack") - } - args = append(args, repoName) - if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil { - fmt.Println(string(out)) + if err := export(group, filepath.Join(tempDir, "launch"), repoName, group.Repository+":run", !publish, !publish); err != nil { return err } return nil } -func groupTomlRepository(tempDir, detectImage string) (string, error) { - var buf bytes.Buffer - cmd := exec.Command("docker", "run", "-v", filepath.Join(tempDir, "workspace")+":/workspace:ro", "--entrypoint", "", detectImage, "bash", "-c", "cat $PACK_BP_GROUP_PATH") - cmd.Stdout = &buf - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return "", err - } - - var groupToml struct { - Repository string `toml:"repository"` - } - if _, err := toml.Decode(buf.String(), &groupToml); err != nil { - return "", err - } - - return groupToml.Repository, nil -} - func cacheDir(appDir string) (string, error) { homeDir := os.Getenv("HOME") if runtime.GOOS == "windows" { diff --git a/cmd/pack/main.go b/cmd/pack/main.go index 6d50fc5623..148a306414 100644 --- a/cmd/pack/main.go +++ b/cmd/pack/main.go @@ -10,20 +10,18 @@ import ( func main() { wd, _ := os.Getwd() - var appDir, detectImage, analyzeImage, exportImage string + var appDir, detectImage string var publish bool buildCommand := &cobra.Command{ Use: "build [IMAGE NAME]", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { repoName := args[0] - return pack.Build(appDir, detectImage, analyzeImage, exportImage, repoName, os.Getenv("PACK_HOST_MACHINE_IP"), publish) + return pack.Build(appDir, detectImage, repoName, publish) }, } buildCommand.Flags().StringVarP(&appDir, "path", "p", wd, "path to app dir") buildCommand.Flags().StringVar(&detectImage, "detect-image", "packs/v3:detect", "detect image") - buildCommand.Flags().StringVar(&analyzeImage, "analyze-image", "packs/v3:analyze", "detect image") - buildCommand.Flags().StringVar(&exportImage, "export-image", "packs/v3:export", "detect image") buildCommand.Flags().BoolVarP(&publish, "publish", "r", false, "publish to registry") rootCmd := &cobra.Command{Use: "pack"} diff --git a/exporter.go b/exporter.go new file mode 100644 index 0000000000..37899f7ab5 --- /dev/null +++ b/exporter.go @@ -0,0 +1,59 @@ +package pack + +import ( + "io/ioutil" + "os" + + "github.com/buildpack/lifecycle" + "github.com/buildpack/packs" + "github.com/buildpack/packs/img" +) + +func export(group lifecycle.BuildpackGroup, launchDir, repoName, stackName string, useDaemon, useDaemonStack bool) error { + origImage, err := readImage(repoName, useDaemon) + if err != nil { + return err + } + + stackImage, err := readImage(stackName, useDaemonStack) + if err != nil || stackImage == nil { + return packs.FailErr(err, "get image for", stackName) + } + + var repoStore img.Store + if useDaemon { + repoStore, err = img.NewDaemon(repoName) + } else { + repoStore, err = img.NewRegistry(repoName) + } + if err != nil { + return packs.FailErr(err, "access", repoName) + } + + tmpDir, err := ioutil.TempDir("", "lifecycle.exporter.layer") + if err != nil { + return packs.FailErr(err, "create temp directory") + } + defer os.RemoveAll(tmpDir) + + exporter := &lifecycle.Exporter{ + Buildpacks: group.Buildpacks, + TmpDir: tmpDir, + Out: os.Stdout, + Err: os.Stderr, + } + newImage, err := exporter.Export( + launchDir, + stackImage, + origImage, + ) + if err != nil { + return packs.FailErrCode(err, packs.CodeFailedBuild) + } + + if err := repoStore.Write(newImage); err != nil { + return packs.FailErrCode(err, packs.CodeFailedUpdate, "write") + } + + return nil +} diff --git a/go.mod b/go.mod index 3240049942..c0970fd697 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,20 @@ module github.com/buildpack/pack require ( github.com/BurntSushi/toml v0.3.0 - github.com/inconshreveable/mousetrap v1.0.0 - github.com/sclevine/spec v1.0.0 + github.com/Microsoft/go-winio v0.4.9 + github.com/buildpack/lifecycle v0.0.0-20180820122535-fa5968b9c0a6 + github.com/buildpack/packs v0.0.0-20180808181744-27a22e86e2e7 + github.com/docker/distribution v2.6.2+incompatible + github.com/docker/docker v1.13.1 + github.com/docker/go-connections v0.4.0 + github.com/docker/go-units v0.3.3 + github.com/golang/mock v1.1.1 + github.com/google/go-cmp v0.2.0 + github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e + github.com/pkg/errors v0.8.0 + github.com/sclevine/spec v0.0.0-20180404042546-a925ac4bfbc9 github.com/spf13/cobra v0.0.3 - github.com/spf13/pflag v1.0.1 + github.com/spf13/pflag v1.0.2 // indirect + golang.org/x/net v0.0.0-20180611182652-db08ff08e862 + golang.org/x/sys v0.0.0-20180724212812-e072cadbbdc8 ) diff --git a/go.sum b/go.sum index cc42603790..175274bb09 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,42 @@ github.com/BurntSushi/toml v0.3.0 h1:e1/Ivsx3Z0FVTV0NSOv/aVgbUWyQuzj7DDnFblkRvsY= github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.4.9/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/buildpack/lifecycle v0.0.0-20180820122535-fa5968b9c0a6 h1:xKmOZ86KOEXIAMDylRQR5Ns4b3sGgpWt3vL0fxbbhEw= +github.com/buildpack/lifecycle v0.0.0-20180820122535-fa5968b9c0a6/go.mod h1:tKBHRckd8Oq05rG6PDrZnOAINA0NkZxlMmUAOxzweAQ= +github.com/buildpack/packs v0.0.0-20180808181744-27a22e86e2e7 h1:nTd2oUeTXp0sHXrWHuFc0T+bBABCrtnHRttoIi3zbUc= +github.com/buildpack/packs v0.0.0-20180808181744-27a22e86e2e7/go.mod h1:jcquCT5a2gbyJw8WMkfgq36g7yc5VDSABVnCctIUKs8= +github.com/buildpack/packs v0.0.0-20180820154346-0aa8a49ec4b4 h1:UKCftJJ3LYjRq8Gudd5HjCXrTCw5w8k1qyBCj0MXj/o= +github.com/buildpack/packs v0.0.0-20180820154346-0aa8a49ec4b4/go.mod h1:jcquCT5a2gbyJw8WMkfgq36g7yc5VDSABVnCctIUKs8= +github.com/docker/distribution v2.6.2+incompatible h1:4FI6af79dfCS/CYb+RRtkSHw3q1L/bnDjG1PcPZtQhM= +github.com/docker/distribution v2.6.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.13.1 h1:5VBhsO6ckUxB0A8CE5LlUJdXzik9cbEbBTQ/ggeml7M= +github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e h1:r4P1NixrrRg3znfuzgWmGkuBmYGsmAmWK4s7bcUa6ZY= +github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk= +github.com/google/go-containerregistry v0.0.0-20180815195620-3165313d6d3f h1:r0kIiA6uoeBOCVhOI09PTJ1qNGRBxXnVAZqPkTk9QVI= +github.com/google/go-containerregistry v0.0.0-20180815195620-3165313d6d3f/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/sclevine/spec v0.0.0-20180404042546-a925ac4bfbc9 h1:xh7f/httJstXLcqjPfzpsgLoLuB1wFNRPaOE8KYxdoM= +github.com/sclevine/spec v0.0.0-20180404042546-a925ac4bfbc9/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sclevine/spec v1.0.0 h1:ILQ08A/CHCz8GGqivOvI54Hy1U40wwcpkf7WtB1MQfY= github.com/sclevine/spec v1.0.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +golang.org/x/net v0.0.0-20180611182652-db08ff08e862 h1:JZi6BqOZ+iSgmLWe6llhGrNnEnK+YB/MRkStwnEfbqM= +golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180816102801-aaf60122140d h1:211XH5RPVP5tOBkz6xm3/b7KxtjqVf6PYG+evqJpE08= +golang.org/x/net v0.0.0-20180816102801-aaf60122140d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sys v0.0.0-20180724212812-e072cadbbdc8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/utils.go b/utils.go new file mode 100644 index 0000000000..3c0c11f671 --- /dev/null +++ b/utils.go @@ -0,0 +1,55 @@ +package pack + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + + "github.com/BurntSushi/toml" + "github.com/buildpack/lifecycle" + "github.com/buildpack/packs" + "github.com/buildpack/packs/img" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +func groupToml(tempDir, detectImage string) (lifecycle.BuildpackGroup, error) { + var buf bytes.Buffer + cmd := exec.Command("docker", "run", "-v", filepath.Join(tempDir, "workspace")+":/workspace:ro", "--entrypoint", "", detectImage, "bash", "-c", "cat $PACK_BP_GROUP_PATH") + cmd.Stdout = &buf + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return lifecycle.BuildpackGroup{}, err + } + + var group lifecycle.BuildpackGroup + if _, err := toml.Decode(buf.String(), &group); err != nil { + return lifecycle.BuildpackGroup{}, err + } + + return group, nil +} + +func readImage(repoName string, useDaemon bool) (v1.Image, error) { + newRepoStore := img.NewRegistry + if useDaemon { + newRepoStore = img.NewDaemon + } + repoStore, err := newRepoStore(repoName) + if err != nil { + return nil, packs.FailErr(err, "access", repoName) + } + + origImage, err := repoStore.Image() + if err != nil { + // Assume error is due to non-existent image + return nil, nil + } + if _, err := origImage.RawManifest(); err != nil { + // Assume error is due to non-existent image + // This is necessary for registries + return nil, nil + } + + return origImage, nil +}