Skip to content

Commit

Permalink
Uses exporter -dry-run to let exporter generate new layer tars
Browse files Browse the repository at this point in the history 
* When exporter generates layers inside the container it fixes perm problems on windows
* Refactors local AddLayer and Rebase to use docker save/load
* Allows daemon and registry builds to share more code

Signed-off-by: Emily Casey <ecasey@pivotal.io>
Signed-off-by: Dave Goddard <dave@goddard.id.au>
  • Loading branch information
@ekcasey @dgodd
ekcasey authored and dgodd committed Nov 16, 2018
1 parent 1a7c665 commit d584a4a
Show file tree
Hide file tree
Showing 17 changed files with 949 additions and 635 deletions.
31 changes: 21 additions & 10 deletions acceptance/acceptance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,10 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
t.Fatalf("Expected to see image %s in %s", repo, contents)
}

t.Log("run image:", repoName)
h.Run(t, exec.Command("docker", "pull", fmt.Sprintf("%s@%s", repoName, imgSHA)))
defer h.Run(t, exec.Command("docker", "rmi", fmt.Sprintf("%s@%s", repoName, imgSHA)))
h.Run(t, exec.Command("docker", "run", "--name="+containerName, "--rm=true", "-d", "-e", "PORT=8080", "-p", ":8080", fmt.Sprintf("%s@%s", repoName, imgSHA)))
defer h.Run(t, exec.Command("docker", "container", "stop", containerName))
launchPort := fetchHostPort(t, containerName)

time.Sleep(5 * time.Second)
Expand Down Expand Up @@ -267,7 +268,7 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
}, spec.Parallel(), spec.Report(report.Terminal{}))

when("pack rebase", func() {
var repoName, containerName, runBefore, runAfter string
var repoName, containerName, runBefore, runAfter, origID string
var buildAndSetRunImage func(runImage, contents1, contents2 string)
var rootContents1 func() string
it.Before(func() {
Expand Down Expand Up @@ -307,24 +308,30 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
})
it.After(func() {
dockerCli.ContainerKill(context.TODO(), containerName, "SIGKILL")
for _, name := range []string{repoName, runBefore, runAfter} {
dockerCli.ImageRemove(context.TODO(), name, dockertypes.ImageRemoveOptions{Force: true, PruneChildren: true})
}
h.Run(t, exec.Command("docker", "rmi", origID))
h.Run(t, exec.Command("docker", "rmi", repoName))
h.Run(t, exec.Command("docker", "rmi", runBefore))
h.Run(t, exec.Command("docker", "rmi", runAfter))
})

when("run on daemon", func() {
it("rebases", func() {
it.Before(func() {
buildAndSetRunImage(runBefore, "contents-before-1", "contents-before-2")

cmd := exec.Command(pack, "build", repoName, "-p", "testdata/node_app/", "--no-pull")
cmd.Env = append(os.Environ(), "PACK_HOME="+packHome)
h.Run(t, cmd)
origID = h.ImageID(t, repoName)
})

it("rebases", func() {
buildAndSetRunImage(runBefore, "contents-before-1", "contents-before-2")

h.AssertEq(t, rootContents1(), "contents-before-1\n")

buildAndSetRunImage(runAfter, "contents-after-1", "contents-after-2")

cmd = exec.Command(pack, "rebase", repoName, "--no-pull")
cmd := exec.Command(pack, "rebase", repoName, "--no-pull")
cmd.Env = append(os.Environ(), "PACK_HOME="+packHome)
h.Run(t, cmd)

Expand All @@ -337,8 +344,7 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
repoName = "localhost:" + registryPort + "/" + repoName
runBefore = "localhost:" + registryPort + "/" + runBefore
runAfter = "localhost:" + registryPort + "/" + runAfter
})
it("rebases", func() {

buildAndSetRunImage(runBefore, "contents-before-1", "contents-before-2")
h.Run(t, exec.Command("docker", "push", runBefore))

Expand All @@ -347,11 +353,14 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
h.Run(t, cmd)

h.AssertEq(t, rootContents1(), "contents-before-1\n")
origID = h.ImageID(t, repoName)
})

it("rebases", func() {
buildAndSetRunImage(runAfter, "contents-after-1", "contents-after-2")
h.Run(t, exec.Command("docker", "push", runAfter))

cmd = exec.Command(pack, "rebase", repoName, "--publish")
cmd := exec.Command(pack, "rebase", repoName, "--publish")
cmd.Env = append(os.Environ(), "PACK_HOME="+packHome)
h.Run(t, cmd)
h.Run(t, exec.Command("docker", "pull", repoName))
Expand Down Expand Up @@ -402,6 +411,7 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
cmd.Env = append(os.Environ(), "PACK_HOME="+packHome)
buildOutput, err := cmd.CombinedOutput()
h.AssertNil(t, err)
defer h.Run(t, exec.Command("docker", "rmi", h.ImageID(t, repoName)))
expectedDetectOutput := "First Mock Buildpack: pass | Second Mock Buildpack: pass | Third Mock Buildpack: pass"
if !strings.Contains(string(buildOutput), expectedDetectOutput) {
t.Fatalf(`Expected build output to contain detection output "%s", got "%s"`, expectedDetectOutput, buildOutput)
Expand Down Expand Up @@ -432,6 +442,7 @@ func testPack(t *testing.T, when spec.G, it spec.S) {
cmd.Env = append(os.Environ(), "PACK_HOME="+packHome)
buildOutput, err = cmd.CombinedOutput()
h.AssertNil(t, err)
defer h.Run(t, exec.Command("docker", "rmi", h.ImageID(t, repoName)))
latestInfo := `No version for 'mock.bp.first' buildpack provided, will use 'mock.bp.first@latest'`
if !strings.Contains(string(buildOutput), latestInfo) {
t.Fatalf(`expected build output to contain "%s", got "%s"`, latestInfo, buildOutput)
Expand Down
217 changes: 192 additions & 25 deletions build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,11 @@ import (
"bytes"
"context"
"crypto/md5"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"os"
"path/filepath"
"strconv"
"strings"
"time"

"github.com/BurntSushi/toml"
"github.com/buildpack/lifecycle"
"github.com/buildpack/lifecycle/img"
"github.com/buildpack/pack/config"
"github.com/buildpack/pack/docker"
"github.com/buildpack/pack/fs"
Expand All @@ -28,6 +20,16 @@ import (
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/uuid"
"github.com/pkg/errors"
"io"
"io/ioutil"
"log"
"math/rand"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)

type BuildFactory struct {
Expand Down Expand Up @@ -58,6 +60,7 @@ type BuildConfig struct {
EnvFile map[string]string
RepoName string
Publish bool
NoPull bool
Buildpacks []string
// Above are copied from BuildFlags are set by init
Cli Docker
Expand Down Expand Up @@ -127,6 +130,7 @@ func (bf *BuildFactory) BuildConfigFromFlags(f *BuildFlags) (*BuildConfig, error
AppDir: appDir,
RepoName: f.RepoName,
Publish: f.Publish,
NoPull: f.NoPull,
Buildpacks: f.Buildpacks,
Cli: bf.Cli,
Stdout: bf.Stdout,
Expand All @@ -139,7 +143,6 @@ func (bf *BuildFactory) BuildConfigFromFlags(f *BuildFlags) (*BuildConfig, error
CacheVolume: fmt.Sprintf("pack-cache-%x", md5.Sum([]byte(appDir))),
}


if f.EnvFile != "" {
b.EnvFile, err = parseEnvFile(f.EnvFile)
if err != nil {
Expand All @@ -155,7 +158,7 @@ func (bf *BuildFactory) BuildConfigFromFlags(f *BuildFlags) (*BuildConfig, error
b.Builder = f.Builder
}
if !f.NoPull {
bf.Log.Printf("Pulling builder image '%s' (use --no-pull flag to skip this step)", f.Builder)
bf.Log.Printf("Pulling builder image '%s' (use --no-pull flag to skip this step)", b.Builder)
if err := bf.Cli.PullImage(b.Builder); err != nil {
return nil, err
}
Expand Down Expand Up @@ -348,7 +351,8 @@ func (b *BuildConfig) Detect() (*lifecycle.BuildpackGroup, error) {
}

tr, errChan := b.FS.CreateTarReader(b.AppDir, filepath.Join(launchDir, "app"), uid, gid)
if err := b.Cli.CopyToContainer(ctx, ctr.ID, "/", tr, dockertypes.CopyToContainerOptions{}); err != nil {
if err := b.Cli.CopyToContainer(ctx, ctr.ID, "/", tr, dockertypes.CopyToContainerOptions{
}); err != nil {
return nil, errors.Wrap(err, "copy app to workspace volume")
}
if err := <-errChan; err != nil {
Expand Down Expand Up @@ -528,34 +532,197 @@ func (b *BuildConfig) tarEnvFile() (io.Reader, error) {
}

func (b *BuildConfig) Export(group *lifecycle.BuildpackGroup) error {
uid, gid, err := b.packUidGid(b.Builder)
ctx := context.Background()
ctr, err := b.Cli.ContainerCreate(ctx, &container.Config{
Image: b.Builder,
Cmd: []string{
"/lifecycle/exporter",
"-dry-run", "/tmp/pack-exporter",
"-image", b.RunImage,
"-launch", launchDir,
"-group", groupPath,
b.RepoName,
},
}, &container.HostConfig{
Binds: []string{
fmt.Sprintf("%s:%s:", b.WorkspaceVolume, launchDir),
},
}, nil, "")
if err != nil {
return errors.Wrap(err, "export")
return errors.Wrap(err, "export container create")
}
defer b.Cli.ContainerRemove(ctx, ctr.ID, dockertypes.ContainerRemoveOptions{})

if err := b.Cli.RunContainer(ctx, ctr.ID, b.Stdout, b.Stderr); err != nil {
return errors.Wrap(err, "run lifecycle/exporter")
}
defer b.Cli.ContainerRemove(ctx, ctr.ID, dockertypes.ContainerRemoveOptions{})

r, _, err := b.Cli.CopyFromContainer(ctx, ctr.ID, "/tmp/pack-exporter")
if err != nil {
return errors.Wrap(err, "copy from exporter container")
}
defer r.Close()

tmpDir, err := ioutil.TempDir("/tmp", "pack.build.")
if err != nil {
return errors.Wrap(err, "tmpdir for exporter")
}
defer os.RemoveAll(tmpDir)

if err := b.FS.Untar(r, tmpDir); err != nil {
return errors.Wrap(err, "untar from exporter container")
}

var imgSHA string
if b.Publish {
localWorkspaceDir, cleanup, err := b.exportVolume(b.Builder, b.WorkspaceVolume)
runImageStore, err := img.NewRegistry(b.RunImage)
if err != nil {
return err
return errors.Wrap(err, "access")
}
runImage, err := runImageStore.Image()
if err != nil {
return errors.Wrap(err, "access")
}
defer cleanup()

imgSHA, err := exportRegistry(group, uid, gid, localWorkspaceDir, b.RepoName, b.RunImage, b.Stdout, b.Stderr)
exporter := &lifecycle.Exporter{
ArtifactsDir: filepath.Join(tmpDir, "pack-exporter"),
Buildpacks: group.Buildpacks,
Out: os.Stdout,
Err: os.Stderr,
}
repoStore, err := img.NewRegistry(b.RepoName)
if err != nil {
return err
return errors.Wrap(err, "access")
}
origImage, err := repoStore.Image()
if err != nil {
return errors.Wrap(err, "access")
}

_, err = origImage.ConfigFile()
if err != nil {
origImage = nil
}
newImage, err := exporter.ExportImage(
launchDir,
launchDir+"/app",
runImage,
origImage,
)
if err != nil {
return errors.Wrap(err, "export to registry")
}
b.Log.Printf("\n*** Image: %s@%s\n", b.RepoName, imgSHA)
if err := repoStore.Write(newImage); err != nil {
return errors.Wrap(err, "write")
}
hash, err := newImage.Digest()
if err != nil {
return errors.Wrap(err, "digest")
}
imgSHA = hash.String()
} else {
var buildpacks []string
for _, b := range group.Buildpacks {
buildpacks = append(buildpacks, b.ID)
var metadata lifecycle.AppImageMetadata
bData, err := ioutil.ReadFile(filepath.Join(tmpDir, "pack-exporter", "metadata.json"))

if err != nil {
return errors.Wrap(err, "read exporter metadata")
}
if err := json.Unmarshal(bData, &metadata); err != nil {
return errors.Wrap(err, "read exporter metadata")
}

if err := exportDaemon(b.Cli, buildpacks, b.WorkspaceVolume, b.RepoName, b.RunImage, b.Stdout, uid, gid); err != nil {
// TODO: move to init
imgFactory, err := image.DefaultFactory()
if err != nil {
return errors.Wrap(err, "create default factory")
}

img, err := imgFactory.NewLocal(b.RunImage, false)
if err != nil {
return errors.Wrap(err, "new local")
}

runImageTopLayer, err := img.TopLayer()
if err != nil {
return errors.Wrap(err, "get run top layer")
}
runImageDigest, err := img.Digest()
if err != nil {
return errors.Wrap(err, "get run digest")
}
metadata.RunImage = lifecycle.RunImageMetadata{
TopLayer: runImageTopLayer,
SHA: runImageDigest,
}

img.Rename(b.RepoName)

var prevMetadata lifecycle.AppImageMetadata
if prevInspect, _, err := b.Cli.ImageInspectWithRaw(context.Background(), b.RepoName); err != nil {
// TODO handle rel error (eg. not prev image not exist)
} else {
label := prevInspect.Config.Labels[lifecycle.MetadataLabel]
if err := json.Unmarshal([]byte(label), &prevMetadata); err != nil {
return errors.Wrap(err, "parsing previous image metadata label")
}
}

// TODO do alpha sort
for index, bp := range metadata.Buildpacks {
var prevBP *lifecycle.BuildpackMetadata
for _, pbp := range prevMetadata.Buildpacks {
if pbp.ID == bp.ID {
prevBP = &pbp
}
}

layerKeys := make([]string, 0, len(bp.Layers))
for n, _ := range bp.Layers {
layerKeys = append(layerKeys, n)
}
sort.Strings(layerKeys)

for _, layerName := range layerKeys {
layer := bp.Layers[layerName]
if layer.SHA == "" {
if prevBP == nil {
return fmt.Errorf("tried to use not exist previous buildpack: %s", bp.ID)
}
// TODO error nicely on not found
layer.SHA = prevBP.Layers[layerName].SHA
if err := img.ReuseLayer(layer.SHA); err != nil {
return errors.Wrapf(err, "reuse layer '%s/%s' from previous image", bp.ID, layerName)
}
metadata.Buildpacks[index].Layers[layerName] = layer
} else {
if err := img.AddLayer(filepath.Join(tmpDir, "pack-exporter", strings.TrimPrefix(layer.SHA, "sha256:")+".tar")); err != nil {
return errors.Wrapf(err, "add layer '%s/%s'", bp.ID, layerName)
}
}
}
}

if err := img.AddLayer(filepath.Join(tmpDir, "pack-exporter", strings.TrimPrefix(metadata.App.SHA, "sha256:")+".tar")); err != nil {
return err
}
if err := img.AddLayer(filepath.Join(tmpDir, "pack-exporter", strings.TrimPrefix(metadata.Config.SHA, "sha256:")+".tar")); err != nil {
return err
}

bData, err = json.Marshal(metadata)
if err != nil {
return errors.Wrap(err, "write exporter metadata")
}
if err := img.SetLabel(lifecycle.MetadataLabel, string(bData)); err != nil {
return errors.Wrap(err, "set image metadata label")
}
if imgSHA, err = img.Save(); err != nil {
return errors.Wrap(err, "save image")
}
}

b.Log.Printf("\n*** Image: %s@%s\n", b.RepoName, imgSHA)
return nil
}

Expand Down
Loading

0 comments on commit d584a4a

Please sign in to comment.