Skip to content

Commit

Permalink
CI: migrate integration-test to GH actions (canonical#1969)
Browse files Browse the repository at this point in the history 
changes:
- drop caching machinery for LXD images and chroots
- divide the workflow in two steps, one for building packages
  and other for running the tests so that the second one can
  be retried in case of error
  • Loading branch information
@aciba90
aciba90 authored Jan 26, 2023
1 parent 94a0049 commit 28b8703
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 105 deletions.
84 changes: 84 additions & 0 deletions .github/workflows/integration.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
name: Integration Tests

on:
pull_request:
branches-ignore:
- 'ubuntu/**'

concurrency:
group: 'ci-${{ github.workflow }}-${{ github.ref }}'
cancel-in-progress: true

defaults:
run:
shell: sh -ex {0}

env:
RELEASE: bionic

jobs:
package-build:
runs-on: ubuntu-22.04
steps:
- name: "Checkout"
uses: actions/checkout@v3
with:
# Fetch all tags for tools/read-version
fetch-depth: 0
- name: Prepare dependencies
run: |
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get -y install \
debhelper \
dh-python \
fakeroot \
python3-setuptools \
sbuild \
ubuntu-dev-tools
sudo sbuild-adduser $USER
cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
- name: Build package
run: |
./packages/bddeb -S -d --release ${{ env.RELEASE }}
sudo -E su $USER -c 'mk-sbuild ${{ env.RELEASE }}'
sudo -E su $USER -c 'DEB_BUILD_OPTIONS=nocheck sbuild --nolog --no-run-lintian --no-run-autopkgtest --verbose --dist=${{ env.RELEASE }} --build-dir=${{ runner.temp }} cloud-init_*.dsc'
- name: Archive debs as artifacts
uses: actions/upload-artifact@v3
with:
name: 'cloud-init-${{ env.RELEASE }}-deb'
path: '${{ runner.temp }}/cloud-init*.deb'
retention-days: 3

integration-tests:
needs: package-build
runs-on: ubuntu-22.04
steps:
- name: "Checkout"
uses: actions/checkout@v3
with:
# Fetch all tags for tools/read-version
fetch-depth: 0
- name: Retrieve cloud-init package
uses: actions/download-artifact@v3
with:
name: 'cloud-init-${{ env.RELEASE }}-deb'
- name: Verify deb package
run: |
ls -hal cloud-init*.deb
- name: Prepare test tools
run: |
sudo DEBIAN_FRONTEND=noninteractive apt-get -y update
sudo DEBIAN_FRONTEND=noninteractive apt-get -y install tox wireguard
- name: Initialize LXD
run: |
ssh-keygen -P "" -q -f ~/.ssh/id_rsa
echo "[lxd]" > /home/$USER/.config/pycloudlib.toml
sudo adduser $USER lxd
# Jammy GH Action runners have docker installed, which edits iptables
# in a way that is incompatible with lxd.
# https://linuxcontainers.org/lxd/docs/master/howto/network_bridge_firewalld/#prevent-issues-with-lxd-and-docker
sudo iptables -I DOCKER-USER -j ACCEPT
sudo lxd init --auto
- name: Run integration Tests
run: |
sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls cloud-init*.deb)" tox -e integration-tests-ci'
105 changes: 0 additions & 105 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,6 @@
language: python
dist: bionic

# We use two different caching strategies. The default is to cache pip
# packages (as most of our jobs use pip packages), which is configured here.
# For the integration tests, we instead want to cache the lxd images and
# package build schroot.
#
# We cache the lxd images because this saves a few seconds in the general
# case, but provides substantial speed-ups when cloud-images.ubuntu.com, the
# source of the images, is under heavy load. The directory in which the lxd
# images are stored (/var/snap/lxd/common/lxd/images/) is not
# readable/writeable by the default user (which is a requirement for caching),
# so we instead cache the `lxd_images/` directory. We move lxd images out of
# there before we run tests and back in once tests are complete. We _move_ the
# images out and only copy the most recent lxd image back into the cache, to
# avoid our cache growing without bound. (We only need the most recent lxd
# image because the integration tests only use a single image.)
#
# We cache the package build schroot because it saves 2-3 minutes per build.
# Without caching, we have to perform a debootstrap for every build. We update
# the schroot before storing it back in the cache, to ensure that we aren't
# just using an increasingly-old schroot as time passes. The cached schroot is
# stored as a tarball, to preserve permissions/ownership.
cache: pip

install:
Expand All @@ -43,90 +22,6 @@ env:
matrix:
fast_finish: true
include:
- name: "Integration Tests"
if: NOT branch =~ /^ubuntu\//
env: {}
cache:
- directories:
- lxd_images
- chroots
before_cache:
- |
# Find the most recent image file
latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)"
# This might be <hash>.rootfs or <hash>, normalise
latest_file="$(basename $latest_file .rootfs)"
# Find all files with that prefix and copy them to our cache dir
sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \;
install:
- git fetch --unshallow
- sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper wireguard
- pip install .
- pip install tox
# bionic has lxd from deb installed, remove it first to ensure
# pylxd talks only to the lxd from snap
- sudo apt remove --purge lxd lxd-client
- sudo rm -Rf /var/lib/lxd
- sudo snap install lxd
- sudo lxd init --auto
- sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles
# Move any cached lxd images into lxd's image dir
- sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \;
- sudo usermod -a -G lxd $USER
- sudo sbuild-adduser $USER
- cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
- echo "[lxd]" > /home/$USER/.config/pycloudlib.toml
script:
# Ubuntu LTS: Build
- ./packages/bddeb -S -d --release bionic
- |
needs_caching=false
if [ -e "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" ]; then
# If we have a cached chroot, move it into place
sudo mkdir -p /var/lib/schroot/chroots/bionic-amd64
sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" -C /var/lib/schroot/chroots/bionic-amd64
# Write its configuration
cat > sbuild-bionic-amd64 << EOM
[bionic-amd64]
description=bionic-amd64
groups=sbuild,root,admin
root-groups=sbuild,root,admin
# Uncomment these lines to allow members of these groups to access
# the -source chroots directly (useful for automated updates, etc).
#source-root-users=sbuild,root,admin
#source-root-groups=sbuild,root,admin
type=directory
profile=sbuild
union-type=overlay
directory=/var/lib/schroot/chroots/bionic-amd64
EOM
sudo mv sbuild-bionic-amd64 /etc/schroot/chroot.d/
sudo chown root /etc/schroot/chroot.d/sbuild-bionic-amd64
# And ensure it's up-to-date.
before_pkgs="$(sudo schroot -c source:bionic-amd64 -d / dpkg -l | sha256sum)"
sudo schroot -c source:bionic-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade"
after_pkgs=$(sudo schroot -c source:bionic-amd64 -d / dpkg -l | sha256sum)
if [ "$before_pkgs" != "$after_pkgs" ]; then
needs_caching=true
fi
else
# Otherwise, create the chroot
sudo -E su $USER -c 'mk-sbuild bionic'
needs_caching=true
fi
# If there are changes to the schroot (or it's entirely new),
# tar up the schroot (to preserve ownership/permissions) and
# move it into the cached dir; no need to compress it because
# Travis will do that anyway
if [ "$needs_caching" = "true" ]; then
sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/bionic-amd64.tar" -C /var/lib/schroot/chroots/bionic-amd64 .
fi
# Use sudo to get a new shell where we're in the sbuild group
# Don't run integration tests when build fails
- |
sudo -E su $USER -c 'DEB_BUILD_OPTIONS=nocheck sbuild --nolog --no-run-lintian --no-run-autopkgtest --verbose --dist=bionic cloud-init_*.dsc' &&
ssh-keygen -P "" -q -f ~/.ssh/id_rsa &&
sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls *.deb)" tox -e integration-tests-ci'
- python: 3.6
env:
TOXENV=lowest-supported
Expand Down

0 comments on commit 28b8703

Please sign in to comment.