Skip to content

Commit

Permalink
prep deb sync workflow
Browse files Browse the repository at this point in the history 
Signed-off-by: Adam Stokes <51892+adam-stokes@users.noreply.github.com>
  • Loading branch information
@adam-stokes
adam-stokes committed Nov 2, 2020
1 parent aa96f76 commit 5bd83a4
Show file tree
Hide file tree
Showing 7 changed files with 81 additions and 48 deletions.
4 changes: 3 additions & 1 deletion cilib/git.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ def clone(url, **subprocess_kwargs):
run(["git", "clone", url], **subprocess_kwargs)


def checkout(ref, new_branch=False, **subprocess_kwargs):
def checkout(ref, new_branch=False, force=False, **subprocess_kwargs):
"""Checkout ref"""
cmd = ["git", "checkout"]
if new_branch:
cmd.append("-b")
if force:
cmd.append("-f")
cmd.append(ref)
run(cmd, **subprocess_kwargs)

Expand Down
6 changes: 4 additions & 2 deletions cilib/models/repos/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ def clone(self, **subprocess_kwargs):
"""Clone package repo"""
git.clone(self.repo, **subprocess_kwargs)

def checkout(self, ref="master", new_branch=False, **subprocess_kwargs):
def checkout(
self, ref="master", new_branch=False, force=False, **subprocess_kwargs
):
"""Checkout ref"""
git.checkout(ref, new_branch, **subprocess_kwargs)
git.checkout(ref, new_branch, force, **subprocess_kwargs)

def commit(self, message, **subprocess_kwargs):
"""Add commit to repo"""
Expand Down
63 changes: 39 additions & 24 deletions cilib/service/deb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,29 +86,44 @@ def sync_debs(self, sign_key):
latest_branch_version = self.deb_model.base.latest_branch_from_major_minor(
_version, exclude_pre
)
if semver.compare(str(latest_branch_version), str(latest_deb_version)) > 0:
if (
not latest_deb_version
or semver.compare(str(latest_branch_version), str(latest_deb_version))
> 0
):
self.log(
f"Found new branch {str(latest_branch_version)} > {str(latest_deb_version)}, building new deb"
)
self.upstream_model.clone()
self.upstream_model.checkout(cwd=self.upstream_model.name)
self.upstream_model.checkout(
ref=f"v{str(latest_branch_version)}",
force=True,
cwd=self.upstream_model.name,
)
with tempfile.TemporaryDirectory() as tmpdir:
click.echo(f"Building {self.deb_model.name} debian package")
self.log(f"Building {self.deb_model.name} debian package")
self.deb_model.base.clone(cwd=tmpdir)
self.deb_model.base.checkout(cwd=f"{tmpdir}/{self.deb_model.name}")
self.bump_revision(cwd=f"{tmpdir}/{self.deb_model.name}")
run(
f"cp -a {tmpdir}/{self.deb_model.name}/* {self.upstream_model.name}/.",
shell=True,
)
self.source(sign_key, include_source, cwd=self.upstream_model.name)
self.deb_model.base.commit(
"Automated Build", cwd=f"{tmpdir}/{self.deb_model.name}"
)
self.deb_model.base.push(cwd=f"{tmpdir}/{self.deb_model.name}")
self.upload(enums.DEB_K8S_TRACK_MAP.get(_version))
self.cleanup_source()
self.cleanup_debian(cwd=self.upstream_model.name)
self.deb_model.base.checkout(
ref=f"v{str(latest_branch_version)}",
force=True,
cwd=f"{tmpdir}/{self.deb_model.name}",
)
self.bump_revision(cwd=f"{tmpdir}/{self.deb_model.name}")
cmd_ok(
f"cp -a {tmpdir}/{self.deb_model.name}/* {self.upstream_model.name}/.",
shell=True,
)
self.source(sign_key, cwd=self.upstream_model.name)
self.deb_model.base.add(
["debian/changelog"], cwd=f"{tmpdir}/{self.deb_model.name}"
)
self.deb_model.base.commit(
"Automated Build", cwd=f"{tmpdir}/{self.deb_model.name}"
)
self.deb_model.base.push(cwd=f"{tmpdir}/{self.deb_model.name}")
self.upload(enums.DEB_K8S_TRACK_MAP.get(_version))
self.cleanup_source()
self.cleanup_debian(cwd=self.upstream_model.name)

else:
self.log(
Expand All @@ -127,21 +142,21 @@ def bump_revision(self, **subprocess_kwargs):
def source(self, sign_key, **subprocess_kwargs):
"""Builds the source deb package"""
cmd = ["dpkg-buildpackage", "-S", f"--sign-key={sign_key}"]
click.echo(f"Building package: {cmd}")
run(cmd, **subprocess_kwargs)
self.log(f"Building package: {cmd}")
cmd_ok(cmd, **subprocess_kwargs)

def cleanup_source(self, **subprocess_kwargs):
run("rm -rf *.changes", shell=True, **subprocess_kwargs)
cmd_ok("rm -rf *.changes", shell=True, **subprocess_kwargs)

def cleanup_debian(self, **subprocess_kwargs):
run(["rm", "-rf", "debian"], **subprocess_kwargs)
cmd_ok(["rm", "-rf", "debian"], **subprocess_kwargs)

@sham
def upload(self, ppa, **subprocess_kwargs):
"""Uploads source packages via dput"""
click.echo("Performing upload")
for changes in list(Path(".").glob("*changes")):
cmd_ok(f"dput {ppa} {str(changes)}", **subprocess_kwargs)
cmd = f"dput {ppa} {str(changes)}"
self.log(cmd)
cmd_ok(cmd, **subprocess_kwargs)

# private

Expand Down
8 changes: 8 additions & 0 deletions jobs/build-debs/debuilder-playbook.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@
- name: upgrade system pip
shell: |
pip3 install pip tox --upgrade
- name: copy lp creds
copy:
src: "{{ lookup('env', 'LPCREDS') }}"
dest: /root/lpcreds.ini
owner: root
group: root
mode: 0600
force: yes
- name: copy ssh creds
copy:
src: "{{ lookup('env', 'CDKBOTSSHCREDS') }}"
Expand Down
4 changes: 3 additions & 1 deletion jobs/build-debs/run.sh.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@
set -x

export GIT_SSH_COMMAND='ssh -i /root/.ssh/cdkbot_rsa -oStrictHostKeyChecking=no'
export LPCREDS=/root/lpcreds.ini
export DEBEMAIL="Kubernetes Engineering <k8s-team-ci@canonical.com>"
K8STEAMCI_GPG_KEY="{{ lookup('env', 'K8STEAMCI_GPG_KEY') }}"
DRY_RUN="{{ lookup('env', 'DRY_RUN') }}"

OPTS="--sign-key $K8STEAMCI_GPG_KEY"

if [ "$DRY_RUN" = "true" ]; then
if [[ $DRY_RUN = "true" ]]; then
OPTS="$OPTS --dry-run"
fi

Expand Down
3 changes: 2 additions & 1 deletion requirements.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,4 +50,5 @@ macaroonbakery>=1.3.1
pathos
prettytable
wheel
ipdb
ipdb
ansible
41 changes: 22 additions & 19 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,35 +6,36 @@
#
aiocontextvars==0.2.2 # via -r requirements.in
aioify==0.3.2 # via -r requirements.in
ansible-base==2.10.2 # via ansible
ansible==2.10.1 # via -r requirements.in
appdirs==1.4.4 # via black, charmcraft
appnope==0.1.0 # via ipython
async-generator==1.10 # via -r requirements.in, asyncio-extras
asyncio-extras==1.3.2 # via -r requirements.in
attrs==20.2.0 # via pytest
autopep8==1.5.4 # via -r requirements.in
awscli==1.18.166 # via -r requirements.in
awscli==1.18.169 # via -r requirements.in
backcall==0.2.0 # via ipython
backoff==1.10.0 # via -r requirements.in
bcrypt==3.2.0 # via paramiko
beautifulsoup4==4.9.3 # via bs4
black==20.8b1 # via -r requirements.in
boto3==1.16.6 # via -r requirements.in
botocore==1.19.6 # via awscli, boto3, s3transfer
boto3==1.16.9 # via -r requirements.in
botocore==1.19.9 # via awscli, boto3, s3transfer
bs4==0.0.1 # via -r requirements.in
canonicalwebteam.snapstoreapi==0.4.4 # via -r requirements.in
certifi==2020.6.20 # via requests
cffi==1.14.3 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests
charmcraft==0.5.0 # via -r requirements.in
charmcraft==0.6.0 # via -r requirements.in
click==7.1.2 # via black, mkdocs, nltk, ogc, ogc-plugins-juju, ogc-plugins-runner
colorama==0.3.9 # via awscli, hunter, ogc
configobj==5.0.6 # via -r requirements.in
contextvars==2.4 # via -r requirements.in
cryptography==3.2.1 # via paramiko
cryptography==3.2.1 # via ansible-base, paramiko, secretstorage
decorator==4.4.2 # via ipython, retry
defusedxml==0.6.0 # via python3-openid
dict-deep==2.0.2 # via ogc
dill==0.3.2 # via multiprocess, pathos
dill==0.3.3 # via multiprocess, pathos
distro==1.5.0 # via lazr.restfulclient
docutils==0.15.2 # via awscli
drypy==1.0.1 # via -r requirements.in
Expand All @@ -51,10 +52,11 @@ iniconfig==1.1.1 # via pytest
invoke==1.4.1 # via -r requirements.in
ipdb==0.13.4 # via -r requirements.in
ipython-genutils==0.2.0 # via traitlets
ipython==7.18.1 # via ipdb
ipython==7.19.0 # via ipdb
jedi==0.17.2 # via ipython
jeepney==0.4.3 # via keyring, secretstorage
jenkins-job-builder==3.6.0 # via -r requirements.in
jinja2==2.11.2 # via -r requirements.in, charmcraft, jenkins-job-builder, mkdocs, ogc
jinja2==2.11.2 # via -r requirements.in, ansible-base, charmcraft, jenkins-job-builder, mkdocs, ogc
jmespath==0.10.0 # via boto3, botocore
joblib==0.17.0 # via nltk
juju==2.8.2 # via -r requirements.in
Expand All @@ -74,7 +76,7 @@ markupsafe==1.1.1 # via jinja2
mccabe==0.6.1 # via flake8
melddict==1.0.1 # via ogc
mkdocs-material-extensions==1.0.1 # via mkdocs-material
mkdocs-material==6.1.0 # via -r requirements.in, mkdocs-material-extensions
mkdocs-material==6.1.2 # via -r requirements.in, mkdocs-material-extensions
mkdocs==1.1.2 # via -r requirements.in, mkdocs-material
module-wrapper==0.2.4 # via aioify
monotonic==1.5 # via fasteners
Expand All @@ -86,7 +88,7 @@ oauthlib==3.1.0 # via lazr.restfulclient
ogc-plugins-juju==1.0.39 # via -r requirements.in
ogc-plugins-runner==1.0.24 # via -r requirements.in
ogc==1.0.2 # via -r requirements.in, ogc-plugins-juju, ogc-plugins-runner
packaging==20.4 # via pytest
packaging==20.4 # via ansible-base, pytest
paramiko==2.7.2 # via juju
parso==0.7.1 # via jedi
pathos==0.2.6 # via -r requirements.in
Expand All @@ -96,8 +98,8 @@ pexpect==4.8.0 # via ipython
pickleshare==0.7.5 # via ipython
pluggy==0.13.1 # via pytest
poetry-version==0.1.5 # via module-wrapper
pox==0.2.8 # via pathos
ppft==1.6.6.2 # via pathos
pox==0.2.9 # via pathos
ppft==1.6.6.3 # via pathos
prettytable==1.0.1 # via -r requirements.in
prometheus-client==0.8.0 # via canonicalwebteam.snapstoreapi
prompt-toolkit==3.0.8 # via ipython
Expand All @@ -119,21 +121,22 @@ pytest-asyncio==0.14.0 # via -r requirements.in
pytest-html==2.1.1 # via -r requirements.in
pytest-metadata==1.10.0 # via pytest-html
pytest-mock==3.3.1 # via -r requirements.in
pytest==6.1.1 # via -r requirements.in, pytest-asyncio, pytest-html, pytest-metadata, pytest-mock
pytest==6.1.2 # via -r requirements.in, pytest-asyncio, pytest-html, pytest-metadata, pytest-mock
python-dateutil==2.8.1 # via botocore, canonicalwebteam.snapstoreapi, charmcraft
python-dotenv==0.10.3 # via -r requirements.in, ogc
python-frontmatter==0.5.0 # via -r requirements.in
python-jenkins==1.7.0 # via jenkins-job-builder
python3-openid==3.2.0 # via canonicalwebteam.snapstoreapi
pytz==2020.1 # via pyrfc3339
pytz==2020.4 # via pyrfc3339
pyyaml-include==1.2 # via ogc
pyyaml==5.3.1 # via -r requirements.in, awscli, charmcraft, jenkins-job-builder, juju, jujubundlelib, mkdocs, ogc, ogc-plugins-juju, python-frontmatter, pyyaml-include
regex==2020.10.23 # via black, nltk
pyyaml==5.3.1 # via -r requirements.in, ansible-base, awscli, charmcraft, jenkins-job-builder, juju, jujubundlelib, mkdocs, ogc, ogc-plugins-juju, python-frontmatter, pyyaml-include
regex==2020.10.28 # via black, nltk
requests-toolbelt==0.9.1 # via charmcraft
requests==2.24.0 # via -r requirements.in, canonicalwebteam.snapstoreapi, charmcraft, macaroonbakery, ogc, python-jenkins, requests-toolbelt, theblues
retry==0.9.2 # via -r requirements.in
rsa==4.5 # via awscli
s3transfer==0.3.3 # via awscli, boto3
secretstorage==3.1.2 # via keyring
semver==2.13.0 # via ogc
sh==1.14.1 # via -r requirements.in, ogc, ogc-plugins-juju, ogc-plugins-runner
six==1.15.0 # via bcrypt, configobj, cryptography, fasteners, jenkins-job-builder, launchpadlib, lazr.restfulclient, livereload, lunr, macaroonbakery, packaging, ppft, protobuf, pymacaroons, pynacl, python-dateutil, python-frontmatter, python-jenkins
Expand All @@ -142,10 +145,10 @@ stevedore==3.2.2 # via jenkins-job-builder
tabulate==0.8.7 # via charmcraft
testresources==2.0.1 # via launchpadlib
theblues==0.5.2 # via -r requirements.in, juju
toml==0.10.1 # via autopep8, black, pytest
toml==0.10.2 # via autopep8, black, pytest
tomlkit==0.5.11 # via poetry-version
toposort==1.5 # via juju
tornado==6.0.4 # via livereload, mkdocs
tornado==6.1 # via livereload, mkdocs
tqdm==4.51.0 # via nltk
traitlets==5.0.5 # via ipython
typed-ast==1.4.1 # via black
Expand Down

0 comments on commit 5bd83a4

Please sign in to comment.