Generate a per package metadata file in /var/lib/dpkg/status.d direct…

…ory for drydock (bazelbuild#239) * Add debian status file for dpgks Code review comments and Fix line length add unit tests code review comments plus docker targets add real example for deb Add tags to skip building target on darwin * Fix ci.json * Add tag to test target too * Add keyword args * Remove actual deb from tests
ixdy · Dec 21, 2017 · f5432b8 · f5432b8
1 parent 3caf72f
commit f5432b8
Show file tree

Hide file tree

Showing 9 changed files with 254 additions and 24 deletions.
diff --git a/container/build_tar.py b/container/build_tar.py
@@ -13,9 +13,11 @@
 # limitations under the License.
 """This tool build tar files from a list of inputs."""
 
+from contextlib import contextmanager
 import os
 import os.path
 import sys
+import re
 import tarfile
 import tempfile
 
@@ -79,6 +81,18 @@ class TarFile(object):
   class DebError(Exception):
     pass
 
+  PKG_NAME_RE = re.compile(r'Package:\s*(?P<pkg_name>\w+).*')
+  DPKG_STATUS_DIR = '/var/lib/dpkg/status.d'
+  PKG_METADATA_FILE = 'control'
+
+  @staticmethod
+  def parse_pkg_name(metadata, filename):
+    pkg_name_match = TarFile.PKG_NAME_RE.match(metadata)
+    if pkg_name_match:
+      return pkg_name_match.group('pkg_name')
+    else:
+      return os.path.basename(os.path.splitext(filename)[0])
+
   def __init__(self, output, directory, compression):
     self.directory = directory
     self.output = output
@@ -178,6 +192,35 @@ def add_link(self, symlink, destination):
     symlink = os.path.normpath(symlink)
     self.tarfile.add_file(symlink, tarfile.SYMTYPE, link=destination)
 
+  @contextmanager
+  def write_temp_file(self, data, suffix='tar', mode='wb'):
+    (_, tmpfile) = tempfile.mkstemp(suffix=suffix)
+    try:
+      with open(tmpfile, mode='wb') as f:
+        f.write(data)
+      yield tmpfile
+    finally:
+      os.remove(tmpfile)
+
+  def add_pkg_metadata(self, metadata_tar, deb):
+    try:
+      with tarfile.open(metadata_tar) as tar:
+        # Metadata is expected to be in a file.
+        control_file_member = filter(lambda f: os.path.basename(f.name) == TarFile.PKG_METADATA_FILE, tar.getmembers())
+        if not control_file_member:
+           raise self.DebError(deb + ' does not Metadata File!')
+        control_file = tar.extractfile(control_file_member[0])
+        metadata = ''.join(control_file.readlines())
+        destination_file = os.path.join(TarFile.DPKG_STATUS_DIR,
+                                        TarFile.parse_pkg_name(metadata, deb))
+        with self.write_temp_file(data=metadata) as metadata_file:
+          self.add_file(metadata_file, destination_file)
+    except (KeyError, TypeError) as e:
+      raise self.DebError(deb + ' contains invalid Metadata! Exeception {0}'.format(e))
+    except Exception as e:
+      raise self.DebError('Unknown Exception {0}. Please report an issue at'
+                          ' github.com/bazelbuild/rules_docker.'.format(e))
+
   def add_deb(self, deb):
     """Extract a debian package in the output tar.
 
@@ -191,18 +234,30 @@ def add_deb(self, deb):
     Raises:
       DebError: if the format of the deb archive is incorrect.
     """
+    pkg_data_found = False
+    pkg_metadata_found = False
     with archive.SimpleArFile(deb) as arfile:
       current = arfile.next()
-      while current and not current.filename.startswith('data.'):
+      while current:
+        parts = current.filename.split(".")
+        name = parts[0]
+        ext = '.'.join(parts[1:])
+        if name == 'data':
+          pkg_data_found = True
+          # Add pkg_data to image tar
+          with self.write_temp_file(suffix=ext, data=current.data) as tmpfile:
+            self.add_tar(tmpfile)
+        elif name == 'control':
+          pkg_metadata_found = True
+          # Add metadata file to image tar
+          with self.write_temp_file(suffix=ext, data=current.data) as tmpfile:
+            self.add_pkg_metadata(metadata_tar=tmpfile, deb=deb)
         current = arfile.next()
-      if not current:
-        raise self.DebError(deb + ' does not contains a data file!')
-      tmpfile = tempfile.mkstemp(suffix=os.path.splitext(current.filename)[-1])
-      with open(tmpfile[1], 'wb') as f:
-        f.write(current.data)
-      self.add_tar(tmpfile[1])
-      os.remove(tmpfile[1])
 
+    if not pkg_data_found:
+      raise self.DebError(deb + ' does not contains a data file!')
+    if not pkg_metadata_found:
+      raise self.DebError(deb + ' does not contains a control file!')
 
 def main(unused_argv):
   # Parse modes arguments

diff --git a/contrib/test.bzl b/contrib/test.bzl
@@ -106,7 +106,7 @@ _container_test = rule(
     implementation = _impl,
 )
 
-def container_test(name, image, configs, driver=None, verbose=None):
+def container_test(name, image, configs, driver=None, verbose=None, **kwargs):
     """A macro to predictably rename the image under test before threading
     it to the container test rule."""
 
@@ -122,12 +122,12 @@ def container_test(name, image, configs, driver=None, verbose=None):
             intermediate_image_name: image,
         }
     )
-
     _container_test(
         name = name,
         image_name = intermediate_image_name,
         image_tar = image_tar_name + ".tar",
         configs = configs,
         verbose = verbose,
         driver = driver,
+        **kwargs
     )
diff --git a/testdata/BUILD b/testdata/BUILD
@@ -21,6 +21,7 @@ load(
     "container_image",
     "container_import",
 )
+load("//testdata:utils.bzl", "generate_deb")
 
 exports_files(["pause.tar"])
 
@@ -346,11 +347,8 @@ py_binary(
     srcs = ["gen_deb.py"],
 )
 
-genrule(
-    name = "generate_deb",
-    outs = ["gen.deb"],
-    cmd = "$(location :gen_deb) $@",
-    tools = [":gen_deb"],
+generate_deb(
+    name = "extras_deb",
 )
 
 # TODO(mattmoor): re-enable once extras.tar works.
@@ -359,7 +357,7 @@ genrule(
 # container_image(
 #     name = "extras_with_deb",
 #     data_path = ".",
-#     debs = [":gen.deb"],
+#     debs = [":extras_deb.deb"],
 #     tars = ["extras.tar"],
 # )
 
@@ -664,3 +662,11 @@ nodejs_image(
     entry_point = "io_bazel_rules_docker/testdata/nodejs_image.js",
     node_modules = "@npm_deps//:node_modules",
 )
+
+generate_deb(
+    name = "pkg1",
+)
+
+generate_deb(
+    name = "pkg2",
+)
diff --git a/testdata/gen_deb.py b/testdata/gen_deb.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """A simple cross-platform helper to create a dummy debian package."""
+import argparse
 from StringIO import StringIO
 import sys
 import tarfile
@@ -30,30 +31,57 @@ def AddArFileEntry(fileobj, filename, content=''):
   if len(content) % 2 != 0:
     fileobj.write('\n')  # 2-byte alignment padding
 
+
+def add_file_to_tar(tar, filename, content):
+   tarinfo = tarfile.TarInfo(filename)
+   tarinfo.size = len(content)
+   tar.addfile(tarinfo, fileobj=StringIO(content))
+
+
+def get_metadata(pkg_name, content=[]):
+  if content:
+     return '\n'.join(content)
+  else:
+     return '\n'.join([
+         'Package: {0}'.format(pkg_name),
+         'Description: Just a dummy description for dummy package {0}'.format(
+             pkg_name),
+     ])
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('-p', action='store', dest='pkg_name',
+                    help='Package name')
+
+parser.add_argument('-a', action='append', dest='metadata',
+                    help='Metadata for package')
+
+parser.add_argument('-o', action='store', dest='outdir',
+                    help='Destination dir')
+
 if __name__ == '__main__':
+  args = parser.parse_args()
+
   # Create data.tar
   tar = StringIO()
   with tarfile.open('data.tar', mode='w', fileobj=tar) as f:
     tarinfo = tarfile.TarInfo('usr/')
     tarinfo.type = tarfile.DIRTYPE
     f.addfile(tarinfo)
-    tarinfo = tarfile.TarInfo('usr/titi')
-    f.addfile(tarinfo, fileobj=StringIO('toto\n'))
+    add_file_to_tar(f, 'usr/{0}'.format(args.pkg_name), 'toto\n')
   data = tar.getvalue()
   tar.close()
   # Create control.tar
   tar = StringIO()
   with tarfile.open('control.tar', mode='w', fileobj=tar) as f:
-    tarinfo = tarfile.TarInfo('control')
-    f.addfile(tarinfo, fileobj=StringIO('\n'.join([
-        'Package: test'
-        'Description: Just a dummy test'
-        ])))
+    metadata_content = get_metadata(pkg_name=args.pkg_name,
+                                    content=args.metadata)
+    add_file_to_tar(f, 'control', metadata_content)
   control = tar.getvalue()
   tar.close()
 
   # Write the final AR archive (the deb package)
-  with open(sys.argv[1], 'w') as f:
+  with open(args.outdir, 'w') as f:
     f.write('!<arch>\n')  # Magic AR header
     AddArFileEntry(f, 'debian-binary', '2.0')
     AddArFileEntry(f, 'control.tar', control)

diff --git a/testdata/utils.bzl b/testdata/utils.bzl
@@ -0,0 +1,27 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility Rules for testing"""
+
+def generate_deb(name, args=[]):
+  args_str = ""
+  if args:
+      args_str = '-a' + ' -a '.join(args)
+  native.genrule(
+    name = name,
+    outs = [name + ".deb"],
+    cmd = "$(location :gen_deb) -p {name} {args_str} -o $@".format(
+        name=name,
+        args_str=args_str),
+    tools = [":gen_deb"],
+  )
diff --git a/tests/container/BUILD b/tests/container/BUILD
@@ -0,0 +1,7 @@
+package(default_visibility = ["//visibility:public"])
+
+py_test(
+    name = "build_tar_test",
+    srcs = ["build_tar_test.py"],
+    deps = ["//container:build_tar"],
+)
diff --git a/tests/container/build_tar_test.py b/tests/container/build_tar_test.py
@@ -0,0 +1,38 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for container build_tar tool"""
+
+from container.build_tar import TarFile
+from contextlib import contextmanager
+import unittest
+
+
+class BuildTarTest(unittest.TestCase):
+
+  def testPackageNameParserValidMetadata(self):
+     metadata = """
+Package: test
+Description: Dummy
+Version: 1.2.4
+"""
+     self.assertEqual('test', TarFile.parse_pkg_name(metadata, "test.deb"))
+
+  def testPackageNameParserInvalidMetadata(self):
+     metadata = "Package Name: Invalid"
+     self.assertEqual('test-invalid-pkg',
+                      TarFile.parse_pkg_name(metadata, "some/path/test-invalid-pkg.deb"))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tests/docker/BUILD b/tests/docker/BUILD
@@ -13,10 +13,27 @@
 # limitations under the License.
 package(default_visibility = ["//visibility:public"])
 
+load("//container:container.bzl", "container_image")
 load("//contrib:test.bzl", "container_test")
 
 container_test(
     name = "structure_test",
     configs = ["//tests/docker/configs:test.yaml"],
     image = "//testdata:link_with_files_base",
 )
+
+container_image(
+    name = "deb_image_with_dpkgs",
+    debs = [
+        "//testdata:pkg1.deb",
+        "//testdata:pkg2.deb",
+        # TODO: (tejaldesai) Once we figure out how to selectively disable tests using build_tag_filters
+        # ":busybox-static_ubuntu1_amd64.deb",
+    ],
+)
+
+container_test(
+    name = "deb_image_with_dpkgs_test",
+    configs = ["//tests/docker/configs:deb_image_with_dpkgs.yaml"],
+    image = ":deb_image_with_dpkgs",
+)