Merge pull request brave#13781 from RyanJarv/fix/cibuild-draft-upload

Clean up upload/publish tools, add initial tests
Crawford30 · Jun 14, 2018 · 875b9ef · 875b9ef
2 parents 081069c + a438b27
commit 875b9ef
Showing 13 changed files with 361 additions and 137 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -5,6 +5,7 @@ dist: trusty
 sudo: required
 group: edge
 before_install:
+  - sudo apt-get install -y python
   - npm i -g npm@5.5.1
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
@@ -28,6 +29,7 @@ env:
     - TEST_DIR=lint
     - TEST_DIR=unit
     - TEST_DIR=codecov
+    - TEST_DIR=tools
     - TEST_DIR=security
     - TEST_DIR=about
     - TEST_DIR=app

diff --git a/package.json b/package.json
@@ -34,7 +34,7 @@
     "start-brk": "node ./tools/start.js --debug-brk=5858 -enable-logging --v=0 --enable-dcheck --user-data-dir-name=brave-development",
     "test": "cross-env NODE_ENV=test mocha \"test/**/*Test.js\"",
     "testsuite": "node ./tools/test.js",
-    "unittest": "cross-env NODE_ENV=test mocha \"test/unit/**/*Test.js\" --globals chrome,DOMParser,XMLSerializer",
+    "unittest": "python tools/test && cross-env NODE_ENV=test mocha \"test/unit/**/*Test.js\" --globals chrome,DOMParser,XMLSerializer",
     "unittest-cov": "node --harmony node_modules/istanbul/lib/cli.js cover ./node_modules/mocha/bin/_mocha --globals chrome,DOMParser,XMLSerializer --report lcovonly --report html --report text -x \"test/unit/**/*Test.js\" -- \"test/unit/**/*Test.js\"",
     "update-pdfjs": "rm -r app/extensions/pdfjs/; cp -r ../pdf.js/build/chromium/ app/extensions/pdfjs/",
     "update-psl": "./tools/updatepsl.sh",

diff --git a/tools/cibuild.py b/tools/cibuild.py
@@ -87,4 +87,4 @@ def run_script(script, args=[]):
 execute([npm, 'run', 'build-package'])
 execute([npm, 'run', 'build-installer'])
 
-run_script('upload.py')
+run_script('upload.py', sys.argv[1:])
diff --git a/tools/lib/github.py b/tools/lib/github.py
@@ -40,7 +40,11 @@ def send(self, method, path, **kw):
       if 'data' in kw:
         kw['data'] = json.dumps(kw['data'])
 
-    r = getattr(requests, method)(url, **kw).json()
+    try:
+      r = getattr(requests, method)(url, **kw).json()
+    except ValueError:
+      # Returned response may be empty in some cases
+      r = {}
     if 'message' in r:
       raise Exception(json.dumps(r, indent=2, separators=(',', ': ')))
     return r

diff --git a/tools/lib/helpers.py b/tools/lib/helpers.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import json
+import traceback
+
+BROWSER_LAPTOP_REPO = 'brave/browser-laptop'
+TARGET_ARCH= os.environ['TARGET_ARCH'] if os.environ.has_key('TARGET_ARCH') else 'x64'
+
+def get_env(env):
+  token = os.environ[env]
+  message = ('Error: Please set the ${} environment variable, which is your personal token'.format(env))
+  assert token, message
+  return token
+
+def release_channel():
+  channel = os.environ['CHANNEL']
+  message = ('Error: Please set the $CHANNEL '
+              'environment variable, which is your release channel')
+  assert channel, message
+  return channel
+
+def get_channel_display_name():
+  d = {'dev': 'Release', 'beta': 'Beta', 'developer': 'Developer', 'nightly': 'Nightly'}
+  return d[release_channel()]
+
+def release_name():
+  return '{0} Channel'.format(get_channel_display_name())
+
+def get_tag():
+  return 'v' + get_version() + release_channel()
+
+def get_tag_without_channel():
+  return 'v' + get_version()
+
+def get_version():
+  return json.load(open('package.json'))['version']
+
+def get_releases_by_tag(repo, tag_name, include_drafts=False):
+  if include_drafts:
+    return [r for r in repo.releases.get() if r['tag_name'] == tag_name]
+  else:
+    return [r for r in repo.releases.get() if r['tag_name'] == tag_name and not r['draft']]
+
+def retry_func(try_func, catch, retries, catch_func=None):
+  for count in range(0, retries + 1):
+    try:
+      ret = try_func(count)
+      break
+    except catch as e:
+      print('[ERROR] Caught exception {}, {} retries left. {}'.format(catch, count, e.message))
+      if catch_func:
+        catch_func(count)
+      if count >= retries:
+        raise e
+  return ret
diff --git a/tools/publish_release.py b/tools/publish_release.py
@@ -1,76 +1,46 @@
 #!/usr/bin/env python
-# for more info see https://developer.github.com/v3/repos/releases/
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
+import sys
+
 from lib.github import GitHub
+from lib.helpers import *
 import requests
 
-BROWSER_LAPTOP_REPO = 'brave/browser-laptop'
-TARGET_ARCH= os.environ['TARGET_ARCH'] if os.environ.has_key('TARGET_ARCH') else 'x64'
 RELEASE_NAME = 'Dev Channel Beta'
 
 def main():
-  github = GitHub(auth_token())
-  releases = github.repos(BROWSER_LAPTOP_REPO).releases.get()
-  version = json.load(open('package.json'))['version']
-  tag = ('v' + version + release_channel())
-  tag_exists = False
-  for release in releases:
-    if not release['draft'] and release['tag_name'] == tag:
-      tag_exists = True
-      break
-  release = create_or_get_release_draft(github, releases, tag,
-                                        tag_exists)
-  # match version to GitHub milestone
-  commit_tag = None
-  parts = version.split('.', 3)
-  if (len(parts) == 3):
-    parts[2] = 'x'
-    commit_tag = '.'.join(parts)
-
-  # Press the publish button.
-  if not tag_exists and commit_tag:
-    publish_release(github, release['id'], tag, commit_tag)
-
-def create_release_draft(github, tag):
-  name = '{0} {1}'.format(RELEASE_NAME, tag)
-  # TODO: Parse release notes from CHANGELOG.md
-  body = '(placeholder)'
-  if body == '':
-    sys.stderr.write('Quit due to empty release note.\n')
-    sys.exit(1)
-  data = dict(tag_name=tag, name=name, body=body, draft=True, prerelease=True)
-  r = github.repos(BROWSER_LAPTOP_REPO).releases.post(data=data)
-  return r
+  repo = GitHub(get_env('GITHUB_TOKEN')).repos(BROWSER_LAPTOP_REPO)
 
-def create_or_get_release_draft(github, releases, tag, tag_exists):
-  # Search for existing draft.
-  for release in releases:
-    if release['draft']:
-      return release
+  release = get_draft(repo, get_tag_without_channel())
+  commit_tag = get_commit_tag(get_version())
 
-  if tag_exists:
-    tag = 'do-not-publish-me'
-  return create_release_draft(github, tag)
+  print("[INFO] Releasing {}".format(release['tag_name']))
+  publish_release(repo, release['id'], get_tag(), commit_tag)
 
-def auth_token():
-  token = os.environ['GITHUB_TOKEN']
-  message = ('Error: Please set the $GITHUB_TOKEN '
-             'environment variable, which is your personal token')
-  assert token, message
-  return token
-
-def release_channel():
-  channel = os.environ['CHANNEL']
-  message = ('Error: Please set the $CHANNEL '
-             'environment variable, which is your release channel')
-  assert channel, message
-  return channel
-
-def publish_release(github, release_id, tag, commit_tag):
+def get_commit_tag(version):
+  parts = get_version().split('.', 3)
+  if (len(parts) == 3):
+    parts[2] = 'x'
+    return '.'.join(parts)
+  else:
+    raise(UserWarning("[ERROR] Invalid version name '%s'", get_version()))
+
+def get_draft(repo, tag):
+  releases = get_releases_by_tag(repo, tag, include_drafts=True)
+  if not releases:
+    raise(UserWarning("[ERROR]: No draft with tag '{}' found, may need to run the ./tools/upload.py script first".format(tag)))
+  elif len(releases) > 1 or not releases[0]['draft'] :
+    raise(UserWarning("[ERROR]: Release with tag {} already exists".format(tag)))
+  return releases[0]
+
+def publish_release(repo, release_id, tag, commit_tag):
   data = dict(draft=False, prerelease=True, tag_name=tag, target_commitish=commit_tag)
-  github.repos(BROWSER_LAPTOP_REPO).releases(release_id).patch(data=data)
+  repo.releases(release_id).patch(data=data)
 
 if __name__ == '__main__':
   import sys

diff --git a/tools/test.js b/tools/test.js
@@ -15,6 +15,9 @@ switch (TEST_DIR) {
   case 'codecov':
     cmd.push('bash tools/codecov.sh')
     break
+  case 'tools':
+    cmd.push('python tools/test')
+    break
   case 'security':
     cmd.push('npm run check-security')
     break

diff --git a/tools/test/__main__.py b/tools/test/__main__.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from test_helpers import *
+from test_publish import *
+from test_upload import *
+
+print unittest.main()
diff --git a/tools/test/mock.py b/tools/test/mock.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class Repo():
+  def __init__(self):
+    self.releases = self.Releases()
+
+  class Releases():
+    def __init__(self):
+      self._releases = []
+    def get(self):
+      return self._releases
diff --git a/tools/test/test_helpers.py b/tools/test/test_helpers.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import unittest
+import os
+
+dirname = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(dirname, '..'))
+from lib.helpers import *
+
+class RetryFunc():
+  def __init__(self):
+    self.ran = 0
+    self.calls = []
+    self.err = UserWarning
+
+  def succeed(self, count):
+    self.ran = self.ran + 1
+    self.calls.append(count)
+
+  def fail(self, count):
+    self.ran = self.ran + 1
+    self.calls.append(count)
+    raise self.err
+
+class TestRetryFunc(unittest.TestCase):
+  def setUp(self):
+    self.retry_func = RetryFunc()
+    self.catch_func = RetryFunc()
+
+  def test_passes_retry_count(self):
+    self.assertRaises(
+      self.retry_func.err,
+      retry_func,
+      self.retry_func.fail,
+      catch=UserWarning, retries=3
+    )
+    self.assertEqual(self.retry_func.calls, [0, 1, 2, 3])
+
+  def test_retries_on_fail(self):
+    self.assertRaises(
+      self.retry_func.err,
+      retry_func,
+      self.retry_func.fail,
+      catch=UserWarning, retries=3
+    )
+    self.assertEqual(self.retry_func.ran, 4)
+
+  def test_run_catch_func_on_fail(self):
+    self.assertRaises(
+      self.retry_func.err,
+      retry_func,
+      self.retry_func.fail,
+      catch_func=self.catch_func.succeed,
+      catch=UserWarning, retries=3
+    )
+    self.assertEqual(self.catch_func.ran, 4)
+
+  def test_no_retry_on_success(self):
+    retry_func(
+      self.retry_func.succeed,
+      catch=UserWarning, retries=3
+    )
+    self.assertEqual(self.retry_func.ran, 1)
+
+  def test_no_run_catch_func_on_success(self):
+    retry_func(
+      self.retry_func.succeed,
+      catch_func=self.catch_func.succeed,
+      catch=UserWarning, retries=3
+    )
+    self.assertEqual(self.catch_func.ran, 0)
+
+if __name__ == '__main__':
+  print unittest.main()
diff --git a/tools/test/test_publish.py b/tools/test/test_publish.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import unittest
+import os
+
+dirname = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(dirname, '..'))
+
+import publish_release
+
+from mock import Repo
+
+class TestPublishGetDraft(unittest.TestCase):
+  def setUp(self):
+    self.repo = Repo()
+
+  def test_fails_on_existing_release(self):
+    self.repo.releases._releases = [{'tag_name': 'test', 'draft': False}]
+    self.assertRaises(UserWarning, publish_release.get_draft, self.repo, 'test')
+
+  def test_fails_on_no_draft(self):
+    self.repo.releases._releases = [{'tag_name': 'old', 'draft': False}]
+    self.assertRaises(UserWarning, publish_release.get_draft, self.repo, 'new')
+
+  def test_succeeds_on_existing_draft(self):
+    self.repo.releases._releases = [{'tag_name': 'test', 'draft': True}]
+    publish_release.get_draft(self.repo, 'test')
+
+if __name__ == '__main__':
+  print unittest.main()
diff --git a/tools/test/test_upload.py b/tools/test/test_upload.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import unittest
+import os
+
+dirname = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(dirname, '..'))
+
+import upload
+
+from mock import Repo
+
+class TestGetDraft(unittest.TestCase):
+  def setUp(self):
+    self.repo = Repo()
+
+  def test_returns_existing_draft(self):
+    self.repo.releases._releases = [{'tag_name': 'test', 'draft': True}]
+    self.assertEquals(upload.get_draft(self.repo, 'test')['tag_name'], 'test')
+
+  def test_fails_on_existing_release(self):
+    self.repo.releases._releases = [{'tag_name': 'test', 'draft': False}]
+    self.assertRaises(UserWarning, upload.get_draft, self.repo, 'test')
+
+  def test_returns_none_on_new_draft(self):
+    self.repo.releases._releases = [{'tag_name': 'old', 'draft': False}]
+    upload.get_draft(self.repo, 'new')
+    self.assertEquals(upload.get_draft(self.repo, 'test'), None)
+
+if __name__ == '__main__':
+  print unittest.main()