Merge branch 'develop' into fix/databricks

cena001plus · Feb 18, 2022 · 4bfa786 · 4bfa786
2 parents f8f40fe + 4c96e97
commit 4bfa786
Show file tree

Hide file tree

Showing 102 changed files with 9,581 additions and 1,592 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!dist
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -7,6 +7,7 @@ on:
       - github-actions-*
     tags:
       - v*
+      - docker-build
   pull_request:
     types: [opened, synchronize]
     branches:
@@ -35,22 +36,17 @@ jobs:
           restore-keys: |
             npm-${{ hashFiles('app/package.json') }}
             npm-
-      - name: Install app dependencies
-        working-directory: app
-        run: yarn install
       - name: Build app
-        working-directory: app
-        run: yarn build
+        run: make app
       - name: Set environment
         env:
           RELEASE_TAG: ${{ github.ref }}
         run: |
           if [[ $RELEASE_TAG =~ ^refs\/tags\/v.*-rc\..*$ ]]; then
              echo "RELEASE_VERSION=$(echo "${{ github.ref }}" | sed "s/^refs\/tags\/v//")" >> $GITHUB_ENV
           fi
-      - name: Build dist
-        run: |
-          python setup.py bdist_wheel sdist
+      - name: Build python
+        run: make python -o app
       - name: Upload dist
         uses: actions/upload-artifact@v2
         with:
@@ -123,8 +119,8 @@ jobs:
       - name: Install fiftyone
         run: |
           pip install -r requirements/extras.txt
-          pip install --pre --extra-index-url https://test.pypi.org/simple/ fiftyone-brain voxel51-eta
           pip install -e .
+          pip install -U --pre --extra-index-url https://test.pypi.org/simple/ fiftyone-brain voxel51-eta
       - name: Install ETA from source
         if: ${{ !startsWith(github.ref, 'refs/heads/rel') && !startsWith(github.ref, 'refs/tags/') }}
         run: |
@@ -196,7 +192,7 @@ jobs:
         uses: actions/download-artifact@v2
         with:
           name: dist
-          path: downloads
+          path: dist
       - name: Install dependencies
         run: |
           pip3 install twine
@@ -216,4 +212,24 @@ jobs:
           TWINE_USERNAME: __token__
           TWINE_NON_INTERACTIVE: 1
         run: |
-          python3 -m twine upload downloads/*
+          python3 -m twine upload dist/*
+
+  build-image:
+    needs: [build, test]
+    if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/tags/docker-build'
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Clone fiftyone
+        uses: actions/checkout@v2
+      - name: Download dist
+        uses: actions/download-artifact@v2
+        with:
+          name: dist
+          path: dist
+      - name: docker
+        run: make docker-export -o python
+      - name: Upload image
+        uses: actions/upload-artifact@v2
+        with:
+          name: docker-image
+          path: fiftyone.tar.gz
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,120 @@
+# Dockerfile for building an image with a source FiftyOne install atop a
+# Debian-based Linux distribution.
+#
+# By default, Ubuntu 20.04 and Python 3.8 are used, but these can be customized
+# via ARGs.
+#
+# ARGs::
+#
+#   BASE_IMAGE (ubuntu:20.04): The Debian-based image to build from
+#   PYTHON_VERSION (3.8): The Python version to install
+#   ROOT_DIR (/fiftyone): The name of the directory within the container that
+#       should be mounted when running
+#
+# Example usage::
+#
+#   # Build
+#   make python
+#   docker build -t voxel51/fiftyone .
+#
+#   # Run
+#   SHARED_DIR=/path/to/shared/dir
+#   docker run \
+#       -v ${SHARED_DIR}:/fiftyone \
+#       -p 5151:5151 \
+#       -it voxel51/fiftyone
+#
+# Copyright 2017-2022, Voxel51, Inc.
+# voxel51.com
+#
+
+# The base image to build from; must be Debian-based (eg Ubuntu)
+ARG BASE_IMAGE=ubuntu:20.04
+FROM $BASE_IMAGE
+
+# The Python version to install
+ARG PYTHON_VERSION=3.8
+
+#
+# Install system packages
+#
+
+RUN apt -y update \
+    && apt -y --no-install-recommends install software-properties-common \
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt -y update \
+    && apt -y upgrade \
+    && apt -y --no-install-recommends install tzdata \
+    && TZ=Etc/UTC \
+    && apt -y --no-install-recommends install \
+        build-essential \
+        ca-certificates \
+        cmake \
+        cmake-data \
+        pkg-config \
+        libcurl4 \
+        libsm6 \
+        libxext6 \
+        libssl-dev \
+        libffi-dev \
+        libxml2-dev \
+        libxslt1-dev \
+        zlib1g-dev \
+        unzip \
+        curl \
+        wget \
+        python${PYTHON_VERSION} \
+        python${PYTHON_VERSION}-dev \
+        python${PYTHON_VERSION}-distutils \
+        ffmpeg \
+    && ln -s /usr/bin/python${PYTHON_VERSION} /usr/local/bin/python \
+    && ln -s /usr/local/lib/python${PYTHON_VERSION} /usr/local/lib/python \
+    && curl https://bootstrap.pypa.io/get-pip.py | python \
+    && rm -rf /var/lib/apt/lists/*
+
+#
+# Install Python dependencies
+#
+# Other packages you might want:
+#   torch torchvision: Torch model training/zoo datasets
+#   tensorflow tensorflow-datasets: TF model training/zoo datasets
+#   pycocotools: COCO-style evaluation
+#   notebook>=5.3 ipywidgets>=7.5: Jupyter notebooks
+#   flash>=0.4: Lightning Flash integration
+#   apache_beam: Apache Beam integration
+#   labelbox: Labelbox integration
+#   shapely: Polyline evaluation
+#   rasterio: GeoTIFF images
+#   pydicom: DICOM images
+#
+
+RUN pip --no-cache-dir install --upgrade pip setuptools wheel ipython
+
+#
+# Install FiftyOne from source
+#
+
+COPY dist dist
+RUN pip --no-cache-dir install dist/*.whl && rm -rf dist
+
+#
+# Configure shared storage
+#
+
+# The name of the shared directory in the container that should be
+# volume-mounted by users to persist data loaded into FiftyOne
+ARG ROOT_DIR=/fiftyone
+
+ENV FIFTYONE_DATABASE_DIR=${ROOT_DIR}/db \
+    FIFTYONE_DEFAULT_DATASET_DIR=${ROOT_DIR}/default \
+    FIFTYONE_DATASET_ZOO_DIR=${ROOT_DIR}/zoo/datasets \
+    FIFTYONE_MODEL_ZOO_DIR=${ROOT_DIR}/zoo/models
+
+#
+# Default behavior
+#
+
+CMD ipython
+
+# Use this if you want the default behavior to instead be to launch the App
+# CMD python /usr/local/lib/python/dist-packages/fiftyone/server/main.py --port 5151
diff --git a/Makefile b/Makefile
@@ -0,0 +1,15 @@
+.PHONY: app python docker docker-export
+
+.DEFAULT_GOAL := docker-export
+
+app:
+	@cd app && yarn && yarn build && cd ..
+
+python: app
+	@python setup.py sdist bdist_wheel
+
+docker: python
+	@docker build -t voxel51/fiftyone .
+
+docker-export: docker
+	@docker save voxel51/fiftyone:latest | gzip > fiftyone.tar.gz
diff --git a/README.md b/README.md
@@ -24,7 +24,7 @@ models**
 [![Downloads](https://pepy.tech/badge/fiftyone)](https://pepy.tech/project/fiftyone)
 [![Build](https://github.com/voxel51/fiftyone/workflows/Build/badge.svg?branch=develop&event=push)](https://github.com/voxel51/fiftyone/actions?query=workflow%3ABuild)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
-[![Slack](https://img.shields.io/badge/Slack-4A154B?logo=slack&logoColor=white)](https://join.slack.com/t/fiftyone-users/shared_invite/zt-gtpmm76o-9AjvzNPBOzevBySKzt02gg)
+[![Slack](https://img.shields.io/badge/Slack-4A154B?logo=slack&logoColor=white)](https://join.slack.com/t/fiftyone-users/shared_invite/zt-s6936w7b-2R5eVPJoUw008wP7miJmPQ)
 [![Medium](https://img.shields.io/badge/Medium-12100E?logo=medium&logoColor=white)](https://medium.com/voxel51)
 [![Mailing list](http://bit.ly/2Md9rxM)](https://share.hsforms.com/1zpJ60ggaQtOoVeBqIZdaaA2ykyk)
 [![Twitter](https://img.shields.io/twitter/follow/Voxel51?style=social)](https://twitter.com/voxel51)
@@ -142,10 +142,10 @@ You will need:
     For example:
 
 ```shell
-# Ubuntu 18.04
+# Ubuntu
 sudo apt install libcurl4 openssl
 
-# Fedora 32
+# Fedora
 sudo dnf install libcurl openssl
 ```
 
@@ -156,14 +156,14 @@ We strongly recommend that you install FiftyOne in a
 to maintain a clean workspace. The install script is only supported in
 POSIX-based systems (e.g. Mac and Linux).
 
-1. Clone the repository:
+First, clone the repository:
 
 ```shell
-git clone --recursive https://github.com/voxel51/fiftyone
+git clone https://github.com/voxel51/fiftyone
 cd fiftyone
 ```
 
-2. Run the installation script:
+Then run the install script:
 
 ```shell
 bash install.bash
@@ -202,7 +202,7 @@ bash install.bash -d
 
 You can install from source in
 [Google Colab](https://colab.research.google.com) by running the following in a
-cell and then **RESTARTING THE RUNTIME**:
+cell and then **restarting the runtime**:
 
 ```shell
 %%shell
@@ -218,6 +218,104 @@ See the
 [docs guide](https://github.com/voxel51/fiftyone/blob/develop/docs/docs_guide.md)
 for information on building and contributing to the documentation.
 
+## Docker installs
+
+Follow the instructions below to build and run a Docker image containing a
+source build of FiftyOne.
+
+### Building an image
+
+First, clone the repository:
+
+```shell
+git clone https://github.com/voxel51/fiftyone
+cd fiftyone
+```
+
+Then build a FiftyOne wheel:
+
+```shell
+make python
+```
+
+and then build the image:
+
+```shell
+docker build -t voxel51/fiftyone .
+```
+
+The default image uses Ubuntu 20.04 and Python 3.8, but you can customize these
+via optional build arguments:
+
+```shell
+docker build \
+    --build-arg BASE_IMAGE=ubuntu:18.04 \
+    --build-arg PYTHON_VERSION=3.9 \
+    -t voxel51/fiftyone .
+```
+
+Refer to the `Dockerfile` itself for additional Python packages that you may
+wish to include in your build.
+
+### Running an image
+
+The image is designed to persist all data in a single `/fityone` directory with
+the following organization:
+
+```
+/fiftyone/
+    db/             # FIFTYONE_DATABASE_DIR
+    default/        # FIFTYONE_DEFAULT_DATASET_DIR
+    zoo/
+        datasets/   # FIFTYONE_DATASET_ZOO_DIR
+        models/     # FIFTYONE_MODEL_ZOO_DIR
+```
+
+Therefore, to run a container, you should mount `/fiftyone` as a local volume
+via `--mount` or `-v`, as shown below:
+
+```shell
+SHARED_DIR=/path/to/shared/dir
+
+docker run -v ${SHARED_DIR}:/fiftyone -p 5151:5151 -it voxel51/fiftyone
+```
+
+The `-p 5151:5151` option is required so that when you
+[launch the App](https://voxel51.com/docs/fiftyone/user_guide/app.html#sessions)
+from within the container you can connect to it at http://localhost:5151 in
+your browser.
+
+You can also include the `-e` or `--env-file` options if you need to further
+[configure FiftyOne](https://voxel51.com/docs/fiftyone/user_guide/config.html).
+
+By default, running the image launches an IPython shell, which you can use as
+normal:
+
+```py
+import fiftyone as fo
+import fiftyone.zoo as foz
+
+dataset = foz.load_zoo_dataset("quickstart")
+session = fo.launch_app(dataset)
+```
+
+Note that any datasets you create inside the Docker image must refer to media
+files within `SHARED_DIR` or another mounted volume if you intend to work with
+datasets between sessions.
+
+### Connecting to a localhost database
+
+If you are using a
+[self-managed database](https://voxel51.com/docs/fiftyone/user_guide/config.html#configuring-a-mongodb-connection)
+that you ordinarily connect to via a URI like `mongodb://localhost`, then you
+will need to tweak this slightly when working in Docker. See
+[this question](https://stackoverflow.com/q/24319662) for details.
+
+On Linux, include `--network="host"` in your `docker run` command and use
+`mongodb://127.0.0.1` for your URI.
+
+On Mac or Windows, use `mongodb://host.docker.internal` for your URI.
+
 ## Uninstallation
 
 You can uninstall FiftyOne as follows:

diff --git a/app/package.json b/app/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@fiftyone/fiftyone",
     "productName": "FiftyOne",
-    "version": "0.19.1",
+    "version": "0.19.2",
     "license": "Apache-2.0",
     "private": true,
     "main": "index.js",

diff --git a/app/packages/app/package.json b/app/packages/app/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@fiftyone/app",
-    "version": "0.19.1",
+    "version": "0.19.2",
     "license": "Apache-2.0",
     "private": true,
     "scripts": {