Rewrite bootstrapper in Python

- This was going to get too complex for bash. Only way to kill those scripts is before they get too complex. - Better progress messages from bootstrapper. - Differentiate between bootstrapper & installer - Cleanup documentation a little bit
wes077 · Jul 2, 2018 · 7158607 · 7158607
1 parent a3cb0e0
commit 7158607
Show file tree

Hide file tree

Showing 10 changed files with 182 additions and 105 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -74,7 +74,7 @@ jobs:
           name: run tljh installer
           command: |
             docker cp . tljh-systemd-ci:/srv/src
-            docker exec -it tljh-systemd-ci bash /srv/src/installer/install.bash
+            docker exec -it tljh-systemd-ci python3 /srv/src/bootstrap/bootstrap.py
 
       - run:
           name: check jupyterhub is up

diff --git a/Dockerfile b/Dockerfile
@@ -21,8 +21,8 @@ RUN systemctl set-default multi-user.target
 STOPSIGNAL SIGRTMIN+3
 
 # Set up image to be useful out of the box for development & CI
-ENV TLJH_INSTALL_PIP_FLAGS="--no-cache-dir -e"
-ENV TLJH_INSTALL_PIP_SPEC=/srv/src
+ENV TLJH_BOOTSTRAP_DEV=yes
+ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src
 ENV PATH=/opt/tljh/hub/bin:${PATH}
 
 CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=journal 3>&1"]
diff --git a/README.rst b/README.rst
@@ -18,14 +18,5 @@ more information.
 Quick Start
 -----------
 
-On a fresh Ubuntu 18.04 server, you can install The Littlest JupyterHub with:
-
-.. code-block:: bash
-
-   curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/installer/install.bash | sudo bash -
-
-This takes 2-5 minutes to run. When completed, you can access your new JupyterHub
-at the public IP of your server (on the default http port 80)!
-
-For more information (including other installation methods), check out the
-`documentation <https://the-littlest-jupyterhub.readthedocs.io>`_.
+Install The Littlest JupyterHub (TLJH) in under 10 minutes by following the
+`quickstart tutorial <http://the-littlest-jupyterhub.readthedocs.io/en/latest/tutorials/quickstart.html>`_.
diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py
@@ -0,0 +1,125 @@
+"""
+Bootstrap an installation of TLJH.
+
+Sets up just enough TLJH environments to invoke tljh.installer.
+
+This script is run as:
+
+    curl <script-url> | sudo python3 -
+
+Constraints:
+  - Be compatible with Python 3.4 (since we support Ubuntu 16.04)
+  - Use stdlib modules only
+"""
+import os
+import subprocess
+import urllib.request
+import contextlib
+import hashlib
+import tempfile
+
+
+def md5_file(fname):
+    """
+    Return md5 of a given filename
+
+    Copied from https://stackoverflow.com/a/3431838
+    """
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+def check_miniconda_version(prefix, version):
+    """
+    Return true if a miniconda install with version exists at prefix
+    """
+    try:
+        return subprocess.check_output([
+            os.path.join(prefix, 'bin', 'conda'),
+            '-V'
+        ]).decode().strip() == 'conda {}'.format(version)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        # Conda doesn't exist, or wrong version
+        return False
+
+
+@contextlib.contextmanager
+def download_miniconda_installer(version):
+    md5sums = {
+        '4.5.4': "a946ea1d0c4a642ddf0c3a26a18bb16d"
+    }
+
+    if version not in md5sums:
+        raise ValueError(
+            'minicondaversion {} not supported. Supported version:'.format(
+                version, ' '.join(md5sums.keys())
+            ))
+
+    with tempfile.NamedTemporaryFile() as f:
+        installer_url = "https://repo.continuum.io/miniconda/Miniconda3-{}-Linux-x86_64.sh".format(version)
+        urllib.request.urlretrieve(installer_url, f.name)
+
+        if md5_file(f.name) != md5sums[version]:
+            raise Exception('md5 hash mismatch! Downloaded file corrupted')
+
+        yield f.name
+
+
+def install_miniconda(installer_path, prefix):
+    subprocess.check_output([
+        '/bin/bash',
+        installer_path,
+        '-u', '-b',
+        '-p', prefix
+    ], stderr=subprocess.STDOUT)
+
+
+def pip_install(prefix, packages, editable=False):
+    flags = '--no-cache-dir --upgrade'
+    if editable:
+        flags += '--editable'
+    subprocess.check_output([
+        os.path.join(prefix, 'bin', 'python3'),
+        '-m', 'pip',
+        'install', '--no-cache-dir',
+    ] + packages)
+
+
+def main():
+    install_prefix = os.environ.get('TLJH_INSTALL_PREFIX', '/opt/tljh')
+    hub_prefix = os.path.join(install_prefix, 'hub')
+    miniconda_version = '4.5.4'
+
+    print('Checking if TLJH is already installed...')
+    if not check_miniconda_version(hub_prefix, miniconda_version):
+        initial_setup = True
+        print('Downloading & setting up hub environment...')
+        with download_miniconda_installer(miniconda_version) as installer_path:
+            install_miniconda(installer_path, hub_prefix)
+        print('Hub environment set up!')
+    else:
+        initial_setup = False
+        print('TLJH is already installed, will try to upgrade')
+
+    if initial_setup:
+        print('Setting up TLJH installer...')
+    else:
+        print('Upgrading TLJH installer...')
+
+    pip_install(hub_prefix, [
+        os.environ.get('TLJH_BOOTSTRAP_PIP_SPEC', 'git+https://github.com/yuvipanda/the-littlest-jupyterhub.git')
+    ], editable=os.environ.get('TLJH_BOOTSTRAP_DEV', 'no') == 'yes')
+
+    print('Starting TLJH installer...')
+    os.execl(
+        os.path.join(hub_prefix, 'bin', 'python3'),
+        os.path.join(hub_prefix, 'bin', 'python3'),
+        '-m',
+        'tljh.installer'
+    )
+
+if __name__ == '__main__':
+    main()
diff --git a/docs/contributing/dev-setup.rst b/docs/contributing/dev-setup.rst
@@ -34,13 +34,13 @@ The easiest & safest way to develop & test TLJH is with `Docker <https://www.doc
 
       sudo docker exec -it tljh-dev /bin/bash
 
-#. Run the installer from inside the container (see step above): 
-   The container image is already set up to default to a ``dev`` install, so 
+#. Run the bootstrapper from inside the container (see step above):
+   The container image is already set up to default to a ``dev`` install, so
    it'll install from your local repo rather than from github.
 
    .. code-block:: console
 
-      bash /srv/src/installer/install.bash
+      python3 /srv/src/bootstrap/bootstrap.py
 
    The primary hub environment will also be in your PATH already for convenience.
 
@@ -51,8 +51,8 @@ The easiest & safest way to develop & test TLJH is with `Docker <https://www.doc
 #. Make some changes to the repository. You can test easily depending on what
    you changed.
 
-   * If you changed the ``installer/install.bash`` script or any of its dependencies,
-     you can test it by running ``bash /srv/src/installer/install.bash``.
+   * If you changed the ``bootstrap/bootstrap.py`` script or any of its dependencies,
+     you can test it by running ``python3 /srv/src/bootstrap/bootstrap.py``.
 
    * If you changed the ``tljh/installer.py`` code (or any of its dependencies),
      you can test it by running ``python3 -m tljh.installer``.

diff --git a/docs/guides/install.rst b/docs/guides/install.rst
@@ -10,7 +10,7 @@ The quick way to install The Littlest JupyterHub (tljh) is:
 
 .. code-block:: bash
 
-   curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/installer/install.bash | sudo bash -
+   curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/bootstrap/bootstrap.py | sudo python3 -
 
 This takes 2-5 minutes to run. When completed, you can access your new JupyterHub
 at the public IP of your server!
@@ -21,22 +21,22 @@ after installation.
 Slightly less quick installation
 --------------------------------
 
-If you can read ``bash`` and are nervous about the previous installation method,
+If you can read ``python3`` and are nervous about the previous installation method,
 you can inspect the installer script before running it.
 
 
 1. Download the installer script
 
    .. code-block:: bash
 
-      curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/installer/install.bash  -o install.bash
+      curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/bootstrap/bootstrap.py -o bootstrap.py
 
 2. Read the install script source using your favorite text editor
 
 3. Run the installer script
 
    .. code-block:: bash
 
-      sudo install.bash
+      sudo python3 bootstrap.py
 
    This should have the exact same effects as the quick installer method.
diff --git a/docs/index.rst b/docs/index.rst
@@ -11,10 +11,10 @@ On a fresh Ubuntu 18.04 server, you can install The Littlest JupyterHub with:
 
 .. code-block:: bash
 
-   curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/installer/install.bash | sudo bash -
+   curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/bootstrap/bootstrap.py | sudo python3 -
 
 This takes 2-5 minutes to run. When completed, you can access your new JupyterHub
-at the public IP of your server!
+at the public IP of your server! Read the :ref:`tutorial_quickstart` next.
 
 If this installation method (``curl <arbitrary-url> | sudo bash -``)
 makes you nervous, check out the :ref:`other installation methods <installation>` we support!

diff --git a/docs/tutorials/quickstart.rst b/docs/tutorials/quickstart.rst
@@ -25,7 +25,7 @@ Step 1: Install the Littlest JupyterHub (TLJH)
 
 .. code-block:: bash
 
-   curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/installer/install.bash | sudo bash -
+   curl https://raw.githubusercontent.com/yuvipanda/the-littlest-jupyterhub/master/bootstrap/bootstrap.py | sudo python3 -
 
 This takes about 1-3 minutes to finish. When completed, you can visit the
 public IP of your server to use your JupyterHub! You can log in with any username

diff --git a/installer/install.bash b/installer/install.bash
diff --git a/tljh/installer.py b/tljh/installer.py
@@ -14,8 +14,7 @@
 
 def ensure_jupyterhub_service(prefix):
     """
-    Ensure JupyterHub & CHP Services are set up properly
-    """
+    Ensure JupyterHub & CHP Services are set up properly """
     with open(os.path.join(HERE, 'systemd-units', 'jupyterhub.service')) as f:
         hub_unit_template = f.read()
 
@@ -66,31 +65,42 @@ def ensure_jupyterhub_package(prefix):
     ])
 
 
-ensure_jupyterhub_package(HUB_ENV_PREFIX)
-ensure_jupyterhub_service(HUB_ENV_PREFIX)
-
-user.ensure_group('jupyterhub-admins')
-user.ensure_group('jupyterhub-users')
-
-with open('/etc/sudoers.d/jupyterhub-admins', 'w') as f:
-    # JupyterHub admins should have full passwordless sudo access
-    f.write('%jupyterhub-admins ALL = (ALL) NOPASSWD: ALL\n')
-    # `sudo -E` should preserve the $PATH we set. This allows
-    # admins in jupyter terminals to do `sudo -E pip install <package>`,
-    # `pip` is in the $PATH we set in jupyterhub_config.py to include the user conda env.
-    f.write('Defaults exempt_group = jupyterhub-admins\n')
-
-conda.ensure_conda_env(USER_ENV_PREFIX)
-conda.ensure_conda_packages(USER_ENV_PREFIX, [
-    # Conda's latest version is on conda much more so than on PyPI.
-    'conda==4.5.4'
-])
-
-conda.ensure_pip_packages(USER_ENV_PREFIX, [
-    # JupyterHub + notebook package are base requirements for user environment
-    'jupyterhub==0.9.0',
-    'notebook==5.5.0',
-    # Install additional notebook frontends!
-    'jupyterlab==0.32.1',
-    'nteract-on-jupyter==1.8.1'
-])
+def main():
+    print("Setting up JupyterHub...")
+    ensure_jupyterhub_package(HUB_ENV_PREFIX)
+    ensure_jupyterhub_service(HUB_ENV_PREFIX)
+
+    print("Setting up system user groups...")
+    user.ensure_group('jupyterhub-admins')
+    user.ensure_group('jupyterhub-users')
+
+    print("Grainting passwordless sudo to JupyterHub admins...")
+    with open('/etc/sudoers.d/jupyterhub-admins', 'w') as f:
+        # JupyterHub admins should have full passwordless sudo access
+        f.write('%jupyterhub-admins ALL = (ALL) NOPASSWD: ALL\n')
+        # `sudo -E` should preserve the $PATH we set. This allows
+        # admins in jupyter terminals to do `sudo -E pip install <package>`,
+        # `pip` is in the $PATH we set in jupyterhub_config.py to include the user conda env.
+        f.write('Defaults exempt_group = jupyterhub-admins\n')
+
+    print("Setting up user environment...")
+    conda.ensure_conda_env(USER_ENV_PREFIX)
+    conda.ensure_conda_packages(USER_ENV_PREFIX, [
+        # Conda's latest version is on conda much more so than on PyPI.
+        'conda==4.5.4'
+    ])
+
+    conda.ensure_pip_packages(USER_ENV_PREFIX, [
+        # JupyterHub + notebook package are base requirements for user environment
+        'jupyterhub==0.9.0',
+        'notebook==5.5.0',
+        # Install additional notebook frontends!
+        'jupyterlab==0.32.1',
+        'nteract-on-jupyter==1.8.1'
+    ])
+
+    print("Done!")
+
+
+if __name__ == '__main__':
+    main()