Skip to content

Commit

Permalink
Fix issue with builder reproducibility
Browse files Browse the repository at this point in the history 
Use consistent timestamps to avoid unnecessary layer changes

Signed-off-by: Javier Romero <jromero@pivotal.io>
Signed-off-by: Andrew Meyer <ameyer@pivotal.io>
  • Loading branch information
@ameyer-pivotal
ameyer-pivotal committed Oct 7, 2019
1 parent 99b3abf commit 134a304
Show file tree
Hide file tree
Showing 9 changed files with 105 additions and 41 deletions.
22 changes: 9 additions & 13 deletions builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,29 +484,29 @@ func (b *Builder) defaultDirsLayer(dest string) (string, error) {
tw := tar.NewWriter(fh)
defer tw.Close()

now := time.Now()
ts := archive.NormalizedDateTime

if err := tw.WriteHeader(b.packOwnedDir(workspaceDir, now)); err != nil {
if err := tw.WriteHeader(b.packOwnedDir(workspaceDir, ts)); err != nil {
return "", errors.Wrapf(err, "creating %s dir in layer", style.Symbol(workspaceDir))
}

if err := tw.WriteHeader(b.packOwnedDir(layersDir, now)); err != nil {
if err := tw.WriteHeader(b.packOwnedDir(layersDir, ts)); err != nil {
return "", errors.Wrapf(err, "creating %s dir in layer", style.Symbol(layersDir))
}

if err := tw.WriteHeader(b.rootOwnedDir(cnbDir, now)); err != nil {
if err := tw.WriteHeader(b.rootOwnedDir(cnbDir, ts)); err != nil {
return "", errors.Wrapf(err, "creating %s dir in layer", style.Symbol(cnbDir))
}

if err := tw.WriteHeader(b.rootOwnedDir(dist.BuildpacksDir, now)); err != nil {
if err := tw.WriteHeader(b.rootOwnedDir(dist.BuildpacksDir, ts)); err != nil {
return "", errors.Wrapf(err, "creating %s dir in layer", style.Symbol(dist.BuildpacksDir))
}

if err := tw.WriteHeader(b.rootOwnedDir(platformDir, now)); err != nil {
if err := tw.WriteHeader(b.rootOwnedDir(platformDir, ts)); err != nil {
return "", errors.Wrapf(err, "creating %s dir in layer", style.Symbol(platformDir))
}

if err := tw.WriteHeader(b.rootOwnedDir(platformDir+"/env", now)); err != nil {
if err := tw.WriteHeader(b.rootOwnedDir(platformDir+"/env", ts)); err != nil {
return "", errors.Wrapf(err, "creating %s dir in layer", style.Symbol(platformDir+"/env"))
}

Expand Down Expand Up @@ -627,14 +627,12 @@ func (b *Builder) envLayer(dest string, env map[string]string) (string, error) {
tw := tar.NewWriter(fh)
defer tw.Close()

now := time.Now()

for k, v := range env {
if err := tw.WriteHeader(&tar.Header{
Name: path.Join(platformDir, "env", k),
Size: int64(len(v)),
Mode: 0644,
ModTime: now,
ModTime: archive.NormalizedDateTime,
}); err != nil {
return "", err
}
Expand All @@ -656,13 +654,11 @@ func (b *Builder) lifecycleLayer(dest string) (string, error) {
tw := tar.NewWriter(fh)
defer tw.Close()

now := time.Now()

if err := tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeDir,
Name: lifecycleDir,
Mode: 0755,
ModTime: now,
ModTime: archive.NormalizedDateTime,
}); err != nil {
return "", err
}
Expand Down
45 changes: 39 additions & 6 deletions builder/builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"testing"

Expand Down Expand Up @@ -220,6 +221,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
h.IsDirectory(),
h.HasFileMode(0755),
h.HasOwnerAndGroup(1234, 4321),
h.HasModTime(archive.NormalizedDateTime),
)
})

Expand All @@ -233,6 +235,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
h.IsDirectory(),
h.HasOwnerAndGroup(1234, 4321),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)
})

Expand All @@ -246,6 +249,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
h.IsDirectory(),
h.HasOwnerAndGroup(0, 0),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)
})

Expand All @@ -259,6 +263,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
h.IsDirectory(),
h.HasOwnerAndGroup(0, 0),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)
})

Expand All @@ -272,11 +277,13 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
h.IsDirectory(),
h.HasOwnerAndGroup(0, 0),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)
h.AssertOnTarEntry(t, layerTar, "/platform/env",
h.IsDirectory(),
h.HasOwnerAndGroup(0, 0),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)
})

Expand Down Expand Up @@ -487,41 +494,49 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
h.AssertOnTarEntry(t, layerTar, "/cnb/lifecycle",
h.IsDirectory(),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/cnb/lifecycle/detector",
h.ContentEquals("detector"),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/cnb/lifecycle/restorer",
h.ContentEquals("restorer"),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/cnb/lifecycle/analyzer",
h.ContentEquals("analyzer"),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/cnb/lifecycle/builder",
h.ContentEquals("builder"),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/cnb/lifecycle/exporter",
h.ContentEquals("exporter"),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/cnb/lifecycle/cacher",
h.ContentEquals("cacher"),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/cnb/lifecycle/launcher",
h.ContentEquals("launcher"),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)
})

Expand Down Expand Up @@ -738,7 +753,8 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
it("adds the order.toml to the image", func() {
layerTar, err := baseImage.FindLayerWithPath("/cnb/order.toml")
h.AssertNil(t, err)
h.AssertOnTarEntry(t, layerTar, "/cnb/order.toml", h.ContentEquals(`[[order]]
h.AssertOnTarEntry(t, layerTar, "/cnb/order.toml",
h.ContentEquals(`[[order]]
[[order.group]]
id = "buildpack-1-id"
Expand All @@ -748,7 +764,9 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
id = "buildpack-2-id"
version = "buildpack-2-version-1"
optional = true
`))
`),
h.HasModTime(archive.NormalizedDateTime),
)
})

it("adds the order to the order label", func() {
Expand Down Expand Up @@ -799,10 +817,13 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
it("adds the stack.toml to the image", func() {
layerTar, err := baseImage.FindLayerWithPath("/cnb/stack.toml")
h.AssertNil(t, err)
h.AssertOnTarEntry(t, layerTar, "/cnb/stack.toml", h.ContentEquals(`[run-image]
h.AssertOnTarEntry(t, layerTar, "/cnb/stack.toml",
h.ContentEquals(`[run-image]
image = "some/run"
mirrors = ["some/mirror", "other/mirror"]
`))
`),
h.HasModTime(archive.NormalizedDateTime),
)
})

it("adds the stack to the metadata", func() {
Expand Down Expand Up @@ -830,8 +851,14 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) {
it("adds the env vars as files to the image", func() {
layerTar, err := baseImage.FindLayerWithPath("/platform/env/SOME_KEY")
h.AssertNil(t, err)
h.AssertOnTarEntry(t, layerTar, "/platform/env/SOME_KEY", h.ContentEquals(`some-val`))
h.AssertOnTarEntry(t, layerTar, "/platform/env/OTHER_KEY", h.ContentEquals(`other-val`))
h.AssertOnTarEntry(t, layerTar, "/platform/env/SOME_KEY",
h.ContentEquals(`some-val`),
h.HasModTime(archive.NormalizedDateTime),
)
h.AssertOnTarEntry(t, layerTar, "/platform/env/OTHER_KEY",
h.ContentEquals(`other-val`),
h.HasModTime(archive.NormalizedDateTime),
)
})
})
})
Expand Down Expand Up @@ -909,6 +936,12 @@ func assertImageHasBPLayer(t *testing.T, image *fakes.Image, bp dist.Buildpack)

h.AssertOnTarEntry(t, layerTar, dirPath,
h.IsDirectory(),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, path.Dir(dirPath),
h.IsDirectory(),
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, dirPath+"/bin/build",
Expand Down
8 changes: 4 additions & 4 deletions builder/compat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"os"
"path"
"time"

"github.com/BurntSushi/toml"
"github.com/pkg/errors"
Expand Down Expand Up @@ -56,15 +55,15 @@ func compatLifecycle(tw *tar.Writer) error {
}

func (b *Builder) compatBuildpacks(tw *tar.Writer) error {
now := time.Now()
if err := tw.WriteHeader(b.rootOwnedDir(compatBuildpacksDir, now)); err != nil {
ts := archive.NormalizedDateTime
if err := tw.WriteHeader(b.rootOwnedDir(compatBuildpacksDir, ts)); err != nil {
return errors.Wrapf(err, "creating %s dir in layer", style.Symbol(dist.BuildpacksDir))
}
for _, bp := range b.additionalBuildpacks {
descriptor := bp.Descriptor()

compatDir := path.Join(compatBuildpacksDir, descriptor.EscapedID())
if err := tw.WriteHeader(b.rootOwnedDir(compatDir, now)); err != nil {
if err := tw.WriteHeader(b.rootOwnedDir(compatDir, ts)); err != nil {
return errors.Wrapf(err, "creating %s dir in layer", style.Symbol(compatDir))
}
compatLink := path.Join(compatDir, descriptor.Info.Version)
Expand Down Expand Up @@ -97,6 +96,7 @@ func addSymlink(tw *tar.Writer, name, linkName string) error {
Linkname: linkName,
Typeflag: tar.TypeSymlink,
Mode: 0644,
ModTime: archive.NormalizedDateTime,
}); err != nil {
return errors.Wrapf(err, "creating %s symlink", style.Symbol(name))
}
Expand Down
23 changes: 20 additions & 3 deletions builder/compat_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,15 +142,25 @@ func testCompat(t *testing.T, when spec.G, it spec.S) {
layerTar, err = baseImage.FindLayerWithPath("/buildpacks/buildpack-1-id/buildpack-1-version-1")
h.AssertNil(t, err)

h.AssertOnTarEntry(t, layerTar, "/buildpacks/buildpack-1-id",
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/buildpacks/buildpack-1-id/buildpack-1-version-1",
h.SymlinksTo("/cnb/buildpacks/buildpack-1-id/buildpack-1-version-1"),
h.HasModTime(archive.NormalizedDateTime),
)

layerTar, err = baseImage.FindLayerWithPath("/buildpacks/buildpack-2-id/buildpack-2-version-1")
h.AssertNil(t, err)

h.AssertOnTarEntry(t, layerTar, "/buildpacks/buildpack-2-id",
h.HasModTime(archive.NormalizedDateTime),
)

h.AssertOnTarEntry(t, layerTar, "/buildpacks/buildpack-2-id/buildpack-2-version-1",
h.SymlinksTo("/cnb/buildpacks/buildpack-2-id/buildpack-2-version-1"),
h.HasModTime(archive.NormalizedDateTime),
)
})

Expand All @@ -176,6 +186,7 @@ func testCompat(t *testing.T, when spec.G, it spec.S) {
h.IsDirectory(),
h.HasOwnerAndGroup(0, 0),
h.HasFileMode(0755),
h.HasModTime(archive.NormalizedDateTime),
)
})

Expand All @@ -192,10 +203,13 @@ func testCompat(t *testing.T, when spec.G, it spec.S) {
it("adds a compat stack.toml to the image", func() {
layerTar, err := baseImage.FindLayerWithPath("/buildpacks/stack.toml")
h.AssertNil(t, err)
h.AssertOnTarEntry(t, layerTar, "/buildpacks/stack.toml", h.ContentEquals(`[run-image]
h.AssertOnTarEntry(t, layerTar, "/buildpacks/stack.toml",
h.ContentEquals(`[run-image]
image = "some/run"
mirrors = ["some/mirror", "other/mirror"]
`))
`),
h.HasModTime(archive.NormalizedDateTime),
)
})
})

Expand Down Expand Up @@ -308,7 +322,10 @@ func testCompat(t *testing.T, when spec.G, it spec.S) {

layerTar, err := baseImage.FindLayerWithPath("/lifecycle")
h.AssertNil(t, err)
h.AssertOnTarEntry(t, layerTar, "/lifecycle", h.SymlinksTo("/cnb/lifecycle"))
h.AssertOnTarEntry(t, layerTar, "/lifecycle",
h.SymlinksTo("/cnb/lifecycle"),
h.HasModTime(archive.NormalizedDateTime),
)
})
})
}
Expand Down
9 changes: 5 additions & 4 deletions dist/layers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ import (
"os"
"path"
"path/filepath"
"time"

"github.com/pkg/errors"

"github.com/buildpack/pack/internal/archive"
)

const BuildpacksDir = "/cnb/buildpacks"
Expand All @@ -33,13 +34,13 @@ func BuildpackLayer(dest string, uid, gid int, bp Buildpack) (string, error) {
tw := tar.NewWriter(fh)
defer tw.Close()

now := time.Now()
ts := archive.NormalizedDateTime

if err := tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeDir,
Name: path.Join(BuildpacksDir, bpd.EscapedID()),
Mode: 0755,
ModTime: now,
ModTime: ts,
}); err != nil {
return "", err
}
Expand All @@ -49,7 +50,7 @@ func BuildpackLayer(dest string, uid, gid int, bp Buildpack) (string, error) {
Typeflag: tar.TypeDir,
Name: baseTarDir,
Mode: 0755,
ModTime: now,
ModTime: ts,
}); err != nil {
return "", err
}
Expand Down
Loading

0 comments on commit 134a304

Please sign in to comment.