Skip to content

Commit

Permalink
uncouple between layer-id and descriptor-hash (#195)
Browse files Browse the repository at this point in the history
  • Loading branch information
@justadogistaken
justadogistaken authored Jun 18, 2021
1 parent d7f0dc0 commit f807d7b
Show file tree
Hide file tree
Showing 53 changed files with 5,797 additions and 388 deletions.
36 changes: 9 additions & 27 deletions build/local_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
"github.com/alibaba/sealer/parser"
v1 "github.com/alibaba/sealer/types/api/v1"
"github.com/alibaba/sealer/utils"
"github.com/alibaba/sealer/utils/hash"
"github.com/alibaba/sealer/utils/mount"
)

Expand Down Expand Up @@ -163,10 +162,12 @@ func (l *LocalBuilder) ExecBuild() error {
}
}

if layer.Hash == "" {
if layer.ID == "" {
continue
}
baseLayers = append(baseLayers, filepath.Join(common.DefaultLayerDir, layer.Hash.Hex()))

ls := image.NewLayerService()
baseLayers = append(baseLayers, ls.LayerStorage().LayerDataDir(layer.ID))
}
logger.Info("exec all build instructs success !")
return nil
Expand Down Expand Up @@ -272,17 +273,12 @@ func (l *LocalBuilder) execLayer(layer *v1.Layer, tempTarget string) error {
}

func (l *LocalBuilder) calculateLayerHashAndPlaceIt(layer *v1.Layer, tempTarget string) error {
layerHash, err := hash.CheckSumAndPlaceLayer(tempTarget)
layerHash, err := l.LayerStore.RegisterLayerForBuilder(tempTarget)
if err != nil {
return fmt.Errorf("failed to calculate layer hash and place it, err: %v", err)
}

emptyHash := hash.SHA256{}.EmptyDigest()
if layerHash == emptyHash {
layerHash = ""
return fmt.Errorf("failed to register layer, err: %s", err)
}

layer.Hash = layerHash
layer.ID = layerHash
return nil
}

Expand All @@ -292,20 +288,6 @@ func (l *LocalBuilder) UpdateImageMetadata() error {
return err
}
filename := fmt.Sprintf("%s/%s%s", common.DefaultImageMetaRootDir, l.Image.Spec.ID, common.YamlSuffix)
// save layer ids
for _, layer := range l.Image.Spec.Layers {
if layer.Hash != "" {
roLayer, err := store.NewROLayer(layer.Hash, 0)
if err != nil {
return err
}

err = l.LayerStore.RegisterLayerIfNotPresent(roLayer)
if err != nil {
return err
}
}
}
// write image info to its metadata
if err := utils.MarshalYamlToFile(filename, l.Image); err != nil {
return fmt.Errorf("failed to write image yaml:%v", err)
Expand Down Expand Up @@ -418,8 +400,8 @@ func getBaseLayersFromImage(image v1.Image) (res []string, err error) {
}

for _, layer := range layers {
if layer.Hash != "" {
res = append(res, filepath.Join(common.DefaultLayerDir, layer.Hash.Hex()))
if layer.ID != "" {
res = append(res, filepath.Join(common.DefaultLayerDir, layer.ID.Hex()))
}
}
return res, nil
Expand Down
1 change: 0 additions & 1 deletion common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ const (
DefaultImageMetadataFileName = "image_metadata.yaml"
ImageScratch = "scratch"
DefaultImageMetaRootDir = "/var/lib/sealer/metadata"
DefaultLayerDBDir = DefaultImageMetaRootDir + "/layerdb"
DefaultImageMetadataFile = "/var/lib/sealer/metadata/images_metadata.json"
DefaultLayerDir = "/var/lib/sealer/data/overlay2"
TarGzSuffix = ".tar.gz"
Expand Down
4 changes: 3 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ require (
github.com/sealyun/lvscare v1.1.2-alpha.2
github.com/spf13/cobra v1.1.1
github.com/spf13/viper v1.7.0
github.com/vbatts/tar-split v0.11.1
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect
golang.org/x/tools v0.1.2 // indirect
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
gotest.tools v2.2.0+incompatible
k8s.io/api v0.20.4
k8s.io/apimachinery v0.20.4
k8s.io/client-go v0.20.4
Expand Down
11 changes: 2 additions & 9 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/vbatts/tar-split v0.11.1 h1:0Odu65rhcZ3JZaPHxl7tCI3V/C/Q9Zf82UFravl02dE=
github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g=
github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
Expand All @@ -443,7 +445,6 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
Expand Down Expand Up @@ -503,8 +504,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -535,7 +534,6 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I=
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
Expand All @@ -552,7 +550,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down Expand Up @@ -595,8 +592,6 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -659,8 +654,6 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
5 changes: 2 additions & 3 deletions image/default_image.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,6 @@ func (d DefaultImageService) Push(imageName string) error {
if err == nil {
dockerprogress.Message(progressChanOut, "", fmt.Sprintf("Success to Push Image %s", named.CompleteName()))
}
dockerprogress.Message(progressChanOut, "", fmt.Sprintf("Success to Push Image %s", named.CompleteName()))
return err
}

Expand Down Expand Up @@ -247,7 +246,7 @@ func (d DefaultImageService) Delete(imageName string) error {
}

for _, layer := range image.Spec.Layers {
layerID := store.LayerID(layer.Hash)
layerID := store.LayerID(layer.ID)
if isLayerDeletable(layer2ImageNames, layerID) {
err = layerStore.Delete(layerID)
if err != nil {
Expand All @@ -271,7 +270,7 @@ func layer2ImageMap(images []*v1.Image) map[store.LayerID][]string {
var layer2ImageNames = make(map[store.LayerID][]string)
for _, image := range images {
for _, layer := range image.Spec.Layers {
layerID := store.LayerID(layer.Hash)
layerID := store.LayerID(layer.ID)
layer2ImageNames[layerID] = append(layer2ImageNames[layerID], image.Name)
}
}
Expand Down
35 changes: 26 additions & 9 deletions image/default_image_file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
imageutils "github.com/alibaba/sealer/image/utils"
v1 "github.com/alibaba/sealer/types/api/v1"
"github.com/alibaba/sealer/utils"
"github.com/alibaba/sealer/utils/compress"
"github.com/alibaba/sealer/utils/archive"
)

type DefaultImageFileService struct {
Expand Down Expand Up @@ -84,23 +84,25 @@ func (d DefaultImageFileService) save(imageName, imageTar string) error {
if err != nil {
return err
}

pathsToCompress = append(pathsToCompress, layerDirs...)
tempDir, err := utils.MkTmpdir()
if err != nil {
return fmt.Errorf("failed to create %s, err: %v", tempDir, err)
}
defer utils.CleanDir(tempDir)

imageMetadataTempFile := filepath.Join(tempDir, common.DefaultImageMetadataFileName)
repofile := filepath.Join(tempDir, common.DefaultMetadataName)
imageYaml := filepath.Join(common.DefaultImageMetaRootDir, image.Spec.ID+common.YamlSuffix)

ima, err := ioutil.ReadFile(imageYaml)
if err != nil {
return fmt.Errorf("failed to read %s, err: %v", imageYaml, err)
}
if err = ioutil.WriteFile(imageMetadataTempFile, ima, common.FileMode0644); err != nil {
return fmt.Errorf("failed to write temp file %s, err: %v ", imageMetadataTempFile, err)
}

repo, err := json.Marshal(&imageutils.ImageMetadata{ID: image.Spec.ID, Name: named.Raw()})
if err != nil {
return err
Expand All @@ -110,24 +112,37 @@ func (d DefaultImageFileService) save(imageName, imageTar string) error {
}

pathsToCompress = append(pathsToCompress, imageMetadataTempFile, repofile)
_, err = compress.Compress(file, pathsToCompress...)
_, err = archive.TarWithRootDir(file, pathsToCompress...)
return err
}

func (d DefaultImageFileService) load(imageSrc string) (*imageutils.ImageMetadata, error) {
src, err := os.Open(imageSrc)
var (
srcFile *os.File
size int64
err error
image v1.Image
)
srcFile, err = os.Open(imageSrc)
if err != nil {
return nil, fmt.Errorf("failed to open %s, err : %v", imageSrc, err)
}
defer src.Close()
if err = compress.Decompress(src, common.DefaultLayerDir); err != nil {
defer srcFile.Close()

srcFi, err := srcFile.Stat()
if err != nil {
return nil, err
}
size = srcFi.Size()

if _, err = archive.Decompress(srcFile, common.DefaultLayerDir, archive.Options{Compress: false}); err != nil {
return nil, err
}
repofile := filepath.Join(common.DefaultLayerDir, common.DefaultMetadataName)
defer os.Remove(repofile)

imageTempFile := filepath.Join(common.DefaultLayerDir, common.DefaultImageMetadataFileName)
defer os.Remove(imageTempFile)
var image v1.Image

if err = utils.UnmarshalYamlFile(imageTempFile, &image); err != nil {
return nil, fmt.Errorf("failed to parsing %s.yaml, err: %v", imageTempFile, err)
Expand All @@ -139,13 +154,15 @@ func (d DefaultImageFileService) load(imageSrc string) (*imageutils.ImageMetadat
}

for _, layer := range image.Spec.Layers {
if layer.Hash == "" {
if layer.ID == "" {
continue
}
roLayer, err := store.NewROLayer(layer.Hash, 0)
// TODO distributionMetadata
roLayer, err := store.NewROLayer(layer.ID, size, nil)
if err != nil {
return nil, err
}

err = layerStore.RegisterLayerIfNotPresent(roLayer)
if err != nil {
return nil, fmt.Errorf("failed to register layer, err: %v", err)
Expand Down
Loading

0 comments on commit f807d7b

Please sign in to comment.