Run analyze and export directly rather than through docker

Signed-off-by: Dave Goddard <dave@goddard.id.au>
quantumsheep · Aug 21, 2018 · 45e1ae0 · 45e1ae0
1 parent 82a7e97
commit 45e1ae0
Show file tree

Hide file tree

Showing 8 changed files with 222 additions and 112 deletions.
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
@@ -2,8 +2,6 @@ package acceptance_test
 
 import (
 	"bytes"
-	"encoding/hex"
-	"errors"
 	"io/ioutil"
 	"os/exec"
 	"path/filepath"
@@ -42,12 +40,6 @@ func TestPack(t *testing.T) {
 		defer os.RemoveAll(packTmpDir)
 	}
 
-	hostMachineIP, err := findHostMachineIP()
-	if err != nil {
-		panic(err)
-	}
-	os.Setenv("PACK_HOST_MACHINE_IP", hostMachineIP)
-
 	spec.Run(t, "pack", testPack, spec.Report(report.Terminal{}))
 }
 
@@ -64,6 +56,12 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		if err := os.Mkdir(filepath.Join(homeDir, ".docker"), 0777); err != nil {
+			t.Fatal(err)
+		}
+		if err := ioutil.WriteFile(filepath.Join(homeDir, ".docker", "config.json"), []byte("{}"), 0666); err != nil {
+			t.Fatal(err)
+		}
 	})
 	it.After(func() {
 		os.RemoveAll(homeDir)
@@ -141,28 +139,27 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
 		when("'--publish' flag is specified", func() {
 			it.Before(func() {
 				t.Log("push v3/packs:run to local registry")
-				for _, name := range []string{"analyze", "build", "export", "run"} {
-					run(t, exec.Command("docker", "tag", "packs/v3:"+name, fmt.Sprintf("host-machine.local:%s/packs/v3:%s", registryPort, name)))
+				for _, name := range []string{"build", "run"} {
+					run(t, exec.Command("docker", "tag", "packs/v3:"+name, fmt.Sprintf("localhost:%s/packs/v3:%s", registryPort, name)))
 				}
-				run(t, exec.Command("docker", "tag", "packs/v3:run", fmt.Sprintf("localhost:%s/packs/v3:run", registryPort)))
 				run(t, exec.Command("docker", "push", fmt.Sprintf("localhost:%s/packs/v3:run", registryPort)))
 
 				// Build copy of packs/v3:detect with all group repositories pointing to image in local registry
-				cmd := exec.Command("docker", "build", "-t", fmt.Sprintf("host-machine.local:%s/packs/v3:detect", registryPort), "-")
+				cmd := exec.Command("docker", "build", "-t", fmt.Sprintf("localhost:%s/packs/v3:detect", registryPort), "-")
 				cmd.Stdin = bytes.NewReader([]byte(fmt.Sprintf(`
 					FROM packs/v3:detect
 
 					USER root
-					RUN sed -i 's/"packs\/v3"/"host-machine.local:%s\/packs\/v3"/' /buildpacks/order.toml
+					RUN sed -i 's/"packs\/v3"/"localhost:%s\/packs\/v3"/' /buildpacks/order.toml
 
 					USER packs
 				`, registryPort)))
 				run(t, cmd)
 			})
 
 			it.After(func() {
-				for _, name := range []string{"detect", "analyze", "build", "export", "run"} {
-					exec.Command("docker", "rmi", fmt.Sprintf("host-machine.local:%s/packs/v3:%s", registryPort, name)).Run()
+				for _, name := range []string{"detect", "build", "run"} {
+					exec.Command("docker", "rmi", fmt.Sprintf("localhost:%s/packs/v3:%s", registryPort, name)).Run()
 				}
 			})
 
@@ -173,14 +170,13 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
 				t.Log("run pack build")
 				cmd := exec.Command(
 					pack, "build",
-					fmt.Sprintf("host-machine.local:%s/%s", registryPort, repo),
+					fmt.Sprintf("localhost:%s/%s", registryPort, repo),
 					"-p", sourceCodePath,
-					"--detect-image", fmt.Sprintf("host-machine.local:%s/packs/v3:detect", registryPort),
-					"--analyze-image", fmt.Sprintf("host-machine.local:%s/packs/v3:analyze", registryPort),
-					"--export-image", fmt.Sprintf("host-machine.local:%s/packs/v3:export", registryPort),
+					"--detect-image", fmt.Sprintf("localhost:%s/packs/v3:detect", registryPort),
 					"--publish",
 				)
 				cmd.Env = append(os.Environ(), "HOME="+homeDir)
+
 				run(t, cmd)
 
 				t.Log("Checking that registry has contents")
@@ -263,21 +259,3 @@ func assertEq(t *testing.T, actual, expected interface{}) {
 		t.Fatal(diff)
 	}
 }
-
-func findHostMachineIP() (string, error) {
-	txt, err := exec.Command("docker", "run", "ubuntu:18.04", "cat", "/proc/net/route").Output()
-	if err != nil {
-		return "", err
-	}
-	for _, line := range strings.Split(string(txt), "\n") {
-		arr := strings.Split(line, "\t")
-		if len(arr) > 2 && arr[1] == "00000000" {
-			b, err := hex.DecodeString(arr[2])
-			if err != nil {
-				return "", err
-			}
-			return fmt.Sprintf("%d.%d.%d.%d", b[3], b[2], b[1], b[0]), nil
-		}
-	}
-	return "", errors.New("Could not determine host machine ip")
-}
diff --git a/analyzer.go b/analyzer.go
@@ -0,0 +1,35 @@
+package pack
+
+import (
+	"os"
+
+	"github.com/buildpack/lifecycle"
+	"github.com/buildpack/packs"
+)
+
+func analyzer(group lifecycle.BuildpackGroup, launchDir, repoName string, useDaemon bool) error {
+	origImage, err := readImage(repoName, useDaemon)
+	if err != nil {
+		return err
+	}
+
+	if origImage == nil {
+		// no previous image to analyze
+		return nil
+	}
+
+	analyzer := &lifecycle.Analyzer{
+		Buildpacks: group.Buildpacks,
+		Out:        os.Stdout,
+		Err:        os.Stderr,
+	}
+	err = analyzer.Analyze(
+		launchDir,
+		origImage,
+	)
+	if err != nil {
+		return packs.FailErrCode(err, packs.CodeFailedBuild)
+	}
+
+	return nil
+}
diff --git a/build.go b/build.go
@@ -1,7 +1,6 @@
 package pack
 
 import (
-	"bytes"
 	"crypto/md5"
 	"fmt"
 	"io"
@@ -10,11 +9,9 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
-
-	"github.com/BurntSushi/toml"
 )
 
-func Build(appDir, detectImage, analyzeImage, exportImage, repoName, hostMachineIP string, publish bool) error {
+func Build(appDir, detectImage, repoName string, publish bool) error {
 	tempDir, err := ioutil.TempDir("/tmp", "lifecycle.pack.build.")
 	if err != nil {
 		return err
@@ -48,32 +45,13 @@ func Build(appDir, detectImage, analyzeImage, exportImage, repoName, hostMachine
 		return err
 	}
 
-	fmt.Println("*** ANALYZING: Reading information from previous image for possible re-use")
-	// TODO: We assume this will need root to access docker.sock, (if so need to chown afterwards)
-	args := []string{
-		"run",
-		"-v", "/var/run/docker.sock:/var/run/docker.sock",
-		"-v", filepath.Join(tempDir, "docker-config.json") + ":/home/packs/.docker/config.json",
-		"-v", filepath.Join(tempDir, "docker-config.json") + ":/etc/docker/daemon.json",
-		"-v", filepath.Join(tempDir, "launch") + ":/launch",
-		"-v", filepath.Join(tempDir, "workspace") + ":/workspace:ro",
-		analyzeImage,
-	}
-	if hostMachineIP != "" {
-		args = append([]string{args[0], "--add-host", "host-machine.local:" + hostMachineIP}, args[1:]...)
-	}
-	if !publish {
-		args = append(args, "-daemon")
-	}
-	args = append(args, repoName)
-	if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil {
-		fmt.Println(string(out))
+	group, err := groupToml(tempDir, detectImage)
+	if err != nil {
 		return err
 	}
 
-	// Read groupRepoImage from ENV:PACK_BP_GROUP_PATH
-	groupRepoImage, err := groupTomlRepository(tempDir, detectImage)
-	if err != nil {
+	fmt.Println("*** ANALYZING: Reading information from previous image for possible re-use")
+	if err := analyzer(group, filepath.Join(tempDir, "launch"), repoName, !publish); err != nil {
 		return err
 	}
 
@@ -83,7 +61,7 @@ func Build(appDir, detectImage, analyzeImage, exportImage, repoName, hostMachine
 		"-v", filepath.Join(tempDir, "workspace")+":/workspace",
 		"-v", cacheDir+":/cache",
 		"-v", filepath.Join(tempDir, "platform")+":/platform",
-		groupRepoImage+":build",
+		group.Repository+":build",
 	)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -97,52 +75,13 @@ func Build(appDir, detectImage, analyzeImage, exportImage, repoName, hostMachine
 	}
 
 	fmt.Println("*** EXPORTING:")
-	args = []string{
-		"run",
-		"--user", "0",
-		"-v", "/var/run/docker.sock:/var/run/docker.sock",
-		"-v", filepath.Join(tempDir, "docker-config.json") + ":/root/.docker/config.json",
-		"-v", filepath.Join(tempDir, "docker-config.json") + ":/etc/docker/daemon.json",
-		"-v", filepath.Join(tempDir, "launch") + ":/launch:ro",
-		"-v", filepath.Join(tempDir, "workspace") + ":/workspace:ro",
-		exportImage,
-		"-stack", groupRepoImage,
-	}
-	if hostMachineIP != "" {
-		args = append([]string{args[0], "--add-host", "host-machine.local:" + hostMachineIP}, args[1:]...)
-	}
-	if !publish {
-		// TODO: We probably don't want daemon-stack by default
-		args = append(args, "-daemon", "-daemon-stack")
-	}
-	args = append(args, repoName)
-	if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil {
-		fmt.Println(string(out))
+	if err := export(group, filepath.Join(tempDir, "launch"), repoName, group.Repository+":run", !publish, !publish); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func groupTomlRepository(tempDir, detectImage string) (string, error) {
-	var buf bytes.Buffer
-	cmd := exec.Command("docker", "run", "-v", filepath.Join(tempDir, "workspace")+":/workspace:ro", "--entrypoint", "", detectImage, "bash", "-c", "cat $PACK_BP_GROUP_PATH")
-	cmd.Stdout = &buf
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return "", err
-	}
-
-	var groupToml struct {
-		Repository string `toml:"repository"`
-	}
-	if _, err := toml.Decode(buf.String(), &groupToml); err != nil {
-		return "", err
-	}
-
-	return groupToml.Repository, nil
-}
-
 func cacheDir(appDir string) (string, error) {
 	homeDir := os.Getenv("HOME")
 	if runtime.GOOS == "windows" {

diff --git a/cmd/pack/main.go b/cmd/pack/main.go
@@ -10,20 +10,18 @@ import (
 func main() {
 	wd, _ := os.Getwd()
 
-	var appDir, detectImage, analyzeImage, exportImage string
+	var appDir, detectImage string
 	var publish bool
 	buildCommand := &cobra.Command{
 		Use:  "build [IMAGE NAME]",
 		Args: cobra.MinimumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			repoName := args[0]
-			return pack.Build(appDir, detectImage, analyzeImage, exportImage, repoName, os.Getenv("PACK_HOST_MACHINE_IP"), publish)
+			return pack.Build(appDir, detectImage, repoName, publish)
 		},
 	}
 	buildCommand.Flags().StringVarP(&appDir, "path", "p", wd, "path to app dir")
 	buildCommand.Flags().StringVar(&detectImage, "detect-image", "packs/v3:detect", "detect image")
-	buildCommand.Flags().StringVar(&analyzeImage, "analyze-image", "packs/v3:analyze", "detect image")
-	buildCommand.Flags().StringVar(&exportImage, "export-image", "packs/v3:export", "detect image")
 	buildCommand.Flags().BoolVarP(&publish, "publish", "r", false, "publish to registry")
 
 	rootCmd := &cobra.Command{Use: "pack"}

diff --git a/exporter.go b/exporter.go
@@ -0,0 +1,59 @@
+package pack
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/buildpack/lifecycle"
+	"github.com/buildpack/packs"
+	"github.com/buildpack/packs/img"
+)
+
+func export(group lifecycle.BuildpackGroup, launchDir, repoName, stackName string, useDaemon, useDaemonStack bool) error {
+	origImage, err := readImage(repoName, useDaemon)
+	if err != nil {
+		return err
+	}
+
+	stackImage, err := readImage(stackName, useDaemonStack)
+	if err != nil || stackImage == nil {
+		return packs.FailErr(err, "get image for", stackName)
+	}
+
+	var repoStore img.Store
+	if useDaemon {
+		repoStore, err = img.NewDaemon(repoName)
+	} else {
+		repoStore, err = img.NewRegistry(repoName)
+	}
+	if err != nil {
+		return packs.FailErr(err, "access", repoName)
+	}
+
+	tmpDir, err := ioutil.TempDir("", "lifecycle.exporter.layer")
+	if err != nil {
+		return packs.FailErr(err, "create temp directory")
+	}
+	defer os.RemoveAll(tmpDir)
+
+	exporter := &lifecycle.Exporter{
+		Buildpacks: group.Buildpacks,
+		TmpDir:     tmpDir,
+		Out:        os.Stdout,
+		Err:        os.Stderr,
+	}
+	newImage, err := exporter.Export(
+		launchDir,
+		stackImage,
+		origImage,
+	)
+	if err != nil {
+		return packs.FailErrCode(err, packs.CodeFailedBuild)
+	}
+
+	if err := repoStore.Write(newImage); err != nil {
+		return packs.FailErrCode(err, packs.CodeFailedUpdate, "write")
+	}
+
+	return nil
+}
diff --git a/go.mod b/go.mod
@@ -2,8 +2,20 @@ module github.com/buildpack/pack
 
 require (
 	github.com/BurntSushi/toml v0.3.0
-	github.com/inconshreveable/mousetrap v1.0.0
-	github.com/sclevine/spec v1.0.0
+	github.com/Microsoft/go-winio v0.4.9
+	github.com/buildpack/lifecycle v0.0.0-20180820122535-fa5968b9c0a6
+	github.com/buildpack/packs v0.0.0-20180808181744-27a22e86e2e7
+	github.com/docker/distribution v2.6.2+incompatible
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-connections v0.4.0
+	github.com/docker/go-units v0.3.3
+	github.com/golang/mock v1.1.1
+	github.com/google/go-cmp v0.2.0
+	github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e
+	github.com/pkg/errors v0.8.0
+	github.com/sclevine/spec v0.0.0-20180404042546-a925ac4bfbc9
 	github.com/spf13/cobra v0.0.3
-	github.com/spf13/pflag v1.0.1
+	github.com/spf13/pflag v1.0.2 // indirect
+	golang.org/x/net v0.0.0-20180611182652-db08ff08e862
+	golang.org/x/sys v0.0.0-20180724212812-e072cadbbdc8
 )