From 43298f649840c9f46446e0e9ad71c89209d64124 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Thu, 25 Aug 2022 19:40:14 -0400 Subject: [PATCH] Release v0.16.6 (#2008) * bumps, release notes * add to release notes * linting * updating pkg versions * removing unnecessary pull * move App build to end * updating release notes * always migrate when user is admin * moving legacy troubleshooting to another page * adding docs on coordinating migrations * adding helpful error when a dataset fails to load * pinning max requirement * using more cloud-friendly layout * adding upgrade note * update create docs pr action Co-authored-by: brimoor --- .github/workflows/build-docs.yml | 2 +- docs/source/getting_started/install.rst | 21 +--- .../getting_started/troubleshooting.rst | 27 ++++- docs/source/release-notes.rst | 95 ++++++++++++++++++ docs/source/user_guide/config.rst | 98 +++++++++++++++++-- fiftyone/core/dataset.py | 20 ++++ fiftyone/migrations/runner.py | 28 +++--- fiftyone/utils/labelstudio.py | 15 ++- install.bash | 47 +++++---- package/desktop/setup.py | 2 +- requirements/extras.txt | 2 +- setup.py | 6 +- 12 files changed, 287 insertions(+), 76 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 4657e9908d..1c02740c12 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -95,7 +95,7 @@ jobs: cp -r docs-download/ source/docs/fiftyone/ rm -r docs-download/ - name: Create website PR - uses: peter-evans/create-pull-request@v2 + uses: peter-evans/create-pull-request@v4 with: branch: fiftyone-docs commit-message: | diff --git a/docs/source/getting_started/install.rst b/docs/source/getting_started/install.rst index 84cc081889..0efe34c8aa 100644 --- a/docs/source/getting_started/install.rst +++ b/docs/source/getting_started/install.rst @@ -267,7 +267,7 @@ Here's the workflow for downgrading to an older version of FiftyOne: .. code-block:: shell # The version that you wish to downgrade to - VERSION=0.14.4 # for example + VERSION=0.15.1 # Migrate the database fiftyone migrate --all -v $VERSION @@ -283,6 +283,9 @@ your ``fiftyone`` package without first running :ref:`fiftyone migrate `, don't worry, you simply need to reinstall the newer version of FiftyOne and then follow these instructions. +See :ref:`this page ` if you need to install +FiftyOne v0.7.3 or earlier. + .. note:: If you are working with a @@ -290,22 +293,6 @@ reinstall the newer version of FiftyOne and then follow these instructions. can use :ref:`database admin privileges ` to control which clients are allowed to downgrade your FiftyOne deployment. -.. note:: - - The :ref:`fiftyone migrate ` command was introduced - in FiftyOne v0.7.3. If you would like to downgrade from a FiftyOne version - prior to v0.7.3 (to a yet older version), then you will first need to - *upgrade* to v0.7.3 or later and then follow the instructions above. - -.. note:: - - To install a FiftyOne version **prior to v0.7.0**, you must add an - ``--index`` option to ``pip install``: - - .. code-block:: shell - - pip install --index https://pypi.voxel51.com fiftyone== - .. _uninstalling-fiftyone: Uninstalling FiftyOne diff --git a/docs/source/getting_started/troubleshooting.rst b/docs/source/getting_started/troubleshooting.rst index bbb7a5a579..6b5a11246d 100644 --- a/docs/source/getting_started/troubleshooting.rst +++ b/docs/source/getting_started/troubleshooting.rst @@ -165,6 +165,32 @@ then FiftyOne's database service will attempt to start up on import using the MongoDB distribution provided by `fiftyone-db`. If the database fails to start, importing `fiftyone` will result in exceptions being raised. +.. _troubleshooting-downgrades: + +Downgrading to old versions +--------------------------- + +The :ref:`fiftyone migrate ` command was introduced in +FiftyOne v0.7.3. If you would like to downgrade from a FiftyOne version +prior to v0.7.3 (to a yet older version), then you will first need to +:ref:`upgrade ` to v0.7.3 or later and then +:ref:`downgrade `: + +.. code-block:: shell + + # The version that you wish to downgrade to + VERSION=0.7.0 + + pip install fiftyone==0.7.3 + fiftyone migrate --all -v $VERSION + pip install fiftyone==$VERSION + +To install a FiftyOne version prior to v0.7.0, you must add ``--index``: + +.. code-block:: shell + + pip install --index https://pypi.voxel51.com fiftyone== + .. _troubleshooting-mongodb-exits: Database exits @@ -176,7 +202,6 @@ Running `ulimit -n 64000` should resolve the issue. 64,000 is the recommended open file limit. MongoDB has full documentation on the issue `here `_. - .. _troubleshooting-mongodb-linux: Troubleshooting Linux imports diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index ebfe8093d1..2adb2bfbbc 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -3,6 +3,101 @@ FiftyOne Release Notes .. default-role:: code +.. _release-notes-v0.16.6: + +FiftyOne 0.16.6 +--------------- +*Released August 25, 2022* + +App + +- Fixed a bug that caused the App to break when sample tags contained `.` + `#1924 `_ +- Fixed search results alignment + `#1930 `_ +- Fixed App refreshes after view changes had occurred from the view bar + `#1931 `_ +- Fixed mask targets rendering in the tooltip + `#1943 `_ + `#1949 `_ +- Fixed classification confusion matrix connections + `#1967 `_ + +Core + +- Added a save context that enables + :ref:`efficient batch edits ` of datasets and views + `#1727 `_ +- Added Plotly v5 support + `#1981 `_ +- Added a :ref:`quantiles aggregation ` + `#1937 `_ +- Added support for writing transformed images/videos to new locations in the + :func:`transform_images() ` and + :func:`transform_videos() ` functions + `#2007 `_ +- Added support for configuring the + :ref:`package-wide logging level ` + `#2009 `_ +- Added more full-featured support for serializing and deserializing datasets, + views, and samples via `to_dict()` and `from_dict()` + `#1922 `_ +- Added support for dynamic attributes when performing coerced exports + `#1993 `_ +- Introduced the notion of client compatability versions + `#2017 `_ +- Extended :meth:`stats() ` to all + sample collections `#1940 `_ +- Added support for serializing aggregations + `#1911 `_ +- Added :func:`weighted_sample() ` + and :func:`balanced_sample() ` + utility methods `#1925 `_ +- Added an optional ``new_ids=True`` option to + :meth:`Dataset.add_collection() ` + that generates new sample/frame IDs when adding the samples + `#1927 `_ +- Added support for the `path` variable in `dataset.yaml` of + :ref:`YOLOv5 datasets ` + `#1903 `_ +- Fixed a bug that prevented using + :meth:`set_values() ` + to set frame-level label fields + `#1922 `_ +- Fixed automatic declaration of frame fields when computing embeddings on a + frame view `#1922 `_ +- Fixed a regression that caused label ID fields to be returned as + `ObjectID` `#1922 `_ +- Fixed a bug that allowed default frame fields to be excluded + `#1922 `_ +- :class:`ClipsView ` instances will now report + their `metadata` type as |VideoMetadata| + `#1922 `_ +- Fixed + :meth:`load_evaluation_view() ` + when ``select_fields`` is ``True`` + `#1922 `_ +- Fixed boolean field parsing when declaring fields + `#1922 `_ +- Fixed a bug that caused nested embedded documents to corrupt datasets + `#1922 `_ +- Fixed a bug that prevented assignment of array-valued dynamic attributes + to labels `#1922 `_ + +Annotation + +- Added a new :ref:`Label Studio integration! ` + `#1848 `_ +- Optimized loading CVAT annotations and performing operations on + :class:`CVATAnnotationResults ` + `#1944 `_ +- Upgraded the :class:`AnnotationAPI ` + interface `#1997 `_ +- Fixed loading group IDs in CVAT video tasks + `#1917 `_ +- Fixed uploading to a CVAT project when no label schema is provided + `#1926 `_ + .. _release-notes-v0.16.5: FiftyOne 0.16.5 diff --git a/docs/source/user_guide/config.rst b/docs/source/user_guide/config.rst index 9eb2878a2c..2b64a4ff17 100644 --- a/docs/source/user_guide/config.rst +++ b/docs/source/user_guide/config.rst @@ -417,7 +417,7 @@ New FiftyOne versions occasionally introduce data model changes that require database migrations when you :ref:`upgrade ` or :ref:`downgrade `. -Database upgrades happen automatically in two steps: +By default, database upgrades happen automatically in two steps: - **Database**: when you import FiftyOne for the first time using a newer version of the Python package, the database's version is automatically @@ -430,16 +430,26 @@ Database downgrades must be manually performed. See :ref:`this page ` for instructions. You can use the :ref:`fiftyone migrate ` command to view -the current versions of your database and datasets: +the current versions of your client, database, and datasets: .. code-block:: shell - # View your client version - fiftyone --version - - # View your database and dataset versions + # View your client, database, and dataset versions fiftyone migrate --info +.. code-block:: text + + Client version: 0.16.6 + Compatible versions: >=0.16.3,<0.17 + + Database version: 0.16.6 + + dataset version + --------------------------- --------- + bdd100k-validation 0.16.5 + quickstart 0.16.5 + ... + Restricting migrations ~~~~~~~~~~~~~~~~~~~~~~ @@ -448,10 +458,14 @@ allowed to upgrade/downgrade your FiftyOne database. The default is `True`, which means that upgrades are automatically peformed when you connect to your database with newer Python client versions. -If you set `database_admin` to `False`, your database will refuse connections -from your FiftyOne client if its version does not match the database's current -version, and datasets will refuse migrations to versions other than the -database's current version. +If you set `database_admin` to `False`, your client will **never** cause the +database to be migrated to a new version. Instead, you'll see the following +behavior: + +- If your client is compatible with the current database version, you will be + allowed to connect to the database and use FiftyOne +- If your client is not compatible with the current database version, you + will see an informative error message when you import the library You can restrict migrations by adding the following entry to your `~/.fiftyone/config.json` file: @@ -477,6 +491,70 @@ or by setting the following environment variable: trigger automatic database upgrades by connecting to the database with newer Python client versions. +Coordinating a migration +~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are working in an environment where multiple services are connecting to +your MongoDB database at any given time, use this strategy to upgrade your +deployment: + +1. Ensure that all clients are running without database admin privileges, + e.g., by adding this to their `~/.fiftyone/config.json`: + +.. code-block:: json + + { + "database_admin": false + } + +2. Perform a test upgrade of one client and ensure that it is compatible with + your current database version: + +.. code-block:: shell + + # In a test environment + pip install --upgrade fiftyone + + # View client's compatibility info + fiftyone migrate --info + +.. code-block:: python + + import fiftyone as fo + + # Convince yourself that the new client can load a dataset + dataset = fo.load_dataset(...) + +3. Now upgrade the client version used by all services: + +.. code-block:: shell + + # In all client environments + pip install --upgrade fiftyone + +4. Once all services are running the new client version, upgrade the database + with admin privileges: + +.. code-block:: shell + + export FIFTYONE_DATABASE_ADMIN=true + + pip install --upgrade fiftyone + fiftyone migrate --all + +.. note:: + + Newly created datasets will always bear the + :meth:`version ` of the Python + client that created them, which may differ from your database's version + if you are undergoing a migration. + + If the new client's version is not in the compatibility range for the old + clients that are still in use, the old clients will not be able to load + the new datasets. + + Therefore, it is recommended to upgrade all clients as soon as possible! + .. _configuring-timezone: Configuring a timezone diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py index d1099c30fd..b23e0bb0f4 100644 --- a/fiftyone/core/dataset.py +++ b/fiftyone/core/dataset.py @@ -4910,6 +4910,25 @@ def _load_dataset(name, virtual=False): if not virtual: fomi.migrate_dataset_if_necessary(name) + try: + return _do_load_dataset(name, virtual=virtual) + except Exception as e: + try: + version = fomi.get_dataset_revision(name) + except: + raise e + + if version != focn.VERSION: + raise ValueError( + "Failed to load dataset '%s' from v%s using client v%s. " + "You may need to upgrade your client" + % (name, version, focn.VERSION) + ) from e + + raise e + + +def _do_load_dataset(name, virtual=False): try: # pylint: disable=no-member dataset_doc = foo.DatasetDocument.objects.get(name=name) @@ -4968,6 +4987,7 @@ def _load_dataset(name, virtual=False): ] dataset_doc.save() + return dataset_doc, sample_doc_cls, frame_doc_cls diff --git a/fiftyone/migrations/runner.py b/fiftyone/migrations/runner.py index d72a04abb7..72102f033a 100644 --- a/fiftyone/migrations/runner.py +++ b/fiftyone/migrations/runner.py @@ -65,9 +65,9 @@ def migrate_all(destination=None, verbose=False): """Migrates the database and all datasets to the specified destination revision. - If no ``destination`` is provided, the database and each dataset will only - be migrated if their current versions are not compatible with the client's - version. + If ``fiftyone.config.database_admin`` is ``False`` and no ``destination`` + is provided, the database and each dataset will only be migrated if their + current versions are not compatible with the client's version. Args: destination (None): the destination revision. By default, the @@ -85,8 +85,9 @@ def migrate_all(destination=None, verbose=False): def migrate_database_if_necessary(destination=None, verbose=False): """Migrates the database to the specified revision, if necessary. - If no ``destination`` is provided, the database will only be migrated if - its current version is not compatible with the client's version. + If ``fiftyone.config.database_admin`` is ``False`` and no ``destination`` + is provided, the database will only be migrated if its current version is + not compatible with the client's version. Args: destination (None): the destination revision. By default, the @@ -102,7 +103,7 @@ def migrate_database_if_necessary(destination=None, verbose=False): default_destination = destination is None if default_destination: - if _is_compatible_version(head): + if not fo.config.database_admin and _is_compatible_version(head): return destination = foc.VERSION @@ -147,9 +148,9 @@ def needs_migration(name=None, head=None, destination=None): To use this method, specify either the ``name`` of an existing dataset or provide the ``head`` revision of the dataset. - If no ``destination`` is provided, a dataset will always be deemed to - require no migration if its current version if compatible with the client's - version. + If ``fiftyone.config.database_admin`` is ``False`` and no ``destination`` + is provided, a dataset will be deemed to require no migration if its + current version if compatible with the client's version. Args: name (None): the name of the dataset @@ -167,7 +168,7 @@ def needs_migration(name=None, head=None, destination=None): head = "0.0" if destination is None: - if _is_compatible_version(head): + if not fo.config.database_admin and _is_compatible_version(head): return False destination = get_database_revision() @@ -183,8 +184,9 @@ def migrate_dataset_if_necessary(name, destination=None, verbose=False): """Migrates the dataset from its current revision to the specified destination revision. - If no ``destination`` is provided, the dataset will only be migrated if - its current version is not compatible with the client's version. + If ``fiftyone.config.database_admin`` is ``False`` and no ``destination`` + is provided, the dataset will only be migrated if its current version is + not compatible with the client's version. Args: name: the name of the dataset @@ -204,7 +206,7 @@ def migrate_dataset_if_necessary(name, destination=None, verbose=False): default_destination = destination is None if default_destination: - if _is_compatible_version(head): + if not fo.config.database_admin and _is_compatible_version(head): return destination = db_version diff --git a/fiftyone/utils/labelstudio.py b/fiftyone/utils/labelstudio.py index b426f72647..1e95586abf 100644 --- a/fiftyone/utils/labelstudio.py +++ b/fiftyone/utils/labelstudio.py @@ -226,14 +226,19 @@ def _init_project(self, config, samples): def _prepare_tasks(self, samples, label_schema, media_field): """Prepares Label Studio tasks for the given data.""" samples.compute_metadata() + + ids, mime_types, filepaths = samples.values( + ["id", "metadata.mime_type", media_field] + ) + tasks = [ { - "source_id": one.id, - one.media_type: one[media_field], + "source_id": _id, "media_type": "image", - "mime_type": one.metadata.mime_type, + "mime_type": _mime_type, + "image": _filepath, } - for one in samples.select_fields(media_field) + for _id, _mime_type, _filepath in zip(ids, mime_types, filepaths) ] predictions, id_map = {}, {} @@ -244,7 +249,7 @@ def _prepare_tasks(self, samples, label_schema, media_field): smp[label_field], full_result={ "from_name": "label", - "to_name": smp.media_type, + "to_name": "image", "original_width": smp.metadata["width"], "original_height": smp.metadata["height"], "image_rotation": getattr(smp, "rotation", 0), diff --git a/install.bash b/install.bash index 23f082d332..f74db8a934 100644 --- a/install.bash +++ b/install.bash @@ -93,28 +93,6 @@ else pip install fiftyone-db fi -if [ ${BUILD_APP} = true ]; then - echo "***** INSTALLING FIFTYONE-APP *****" - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash - export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm - nvm install ${NODE_VERSION} - nvm use ${NODE_VERSION} - npm -g install yarn - if [ -f ~/.bashrc ]; then - source ~/.bashrc - elif [ -f ~/.bash_profile ]; then - source ~/.bash_profile - else - echo "WARNING: unable to locate a bash profile to 'source'; you may need to start a new shell" - fi - cd app - echo "Building the App. This will take a minute or two..." - yarn install > /dev/null 2>&1 - yarn build - cd .. -fi - if [ ${VOXEL51_INSTALL} = false ]; then echo "***** INSTALLING FIFTYONE-BRAIN *****" pip install fiftyone-brain @@ -138,8 +116,6 @@ if [ ${SOURCE_ETA_INSTALL} = true ]; then git clone https://github.com/voxel51/eta fi cd eta - git checkout develop - git pull if [ ${DEV_INSTALL} = true ] || [ ${VOXEL51_INSTALL} = true ]; then pip install -e . else @@ -152,4 +128,27 @@ if [ ${SOURCE_ETA_INSTALL} = true ]; then cd .. fi +# Do this last since `source` can exit Python virtual environments +if [ ${BUILD_APP} = true ]; then + echo "***** INSTALLING FIFTYONE-APP *****" + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash + export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm + nvm install ${NODE_VERSION} + nvm use ${NODE_VERSION} + npm -g install yarn + if [ -f ~/.bashrc ]; then + source ~/.bashrc + elif [ -f ~/.bash_profile ]; then + source ~/.bash_profile + else + echo "WARNING: unable to locate a bash profile to 'source'; you may need to start a new shell" + fi + cd app + echo "Building the App. This will take a minute or two..." + yarn install > /dev/null 2>&1 + yarn build + cd .. +fi + echo "***** INSTALLATION COMPLETE *****" diff --git a/package/desktop/setup.py b/package/desktop/setup.py index 477fa674af..4d22d26222 100644 --- a/package/desktop/setup.py +++ b/package/desktop/setup.py @@ -16,7 +16,7 @@ import shutil -VERSION = "0.22.1" +VERSION = "0.22.2" def get_version(): diff --git a/requirements/extras.txt b/requirements/extras.txt index 0c05afbfa2..a08e34b7a3 100644 --- a/requirements/extras.txt +++ b/requirements/extras.txt @@ -2,7 +2,7 @@ apache-beam>=2.33.0 google-api-python-client>=1.6.5 google-cloud-storage>=1.36 httplib2<=0.15 -ipywidgets>=7.5 +ipywidgets>=7.5,<8 lightning-flash>=0.4.0 notebook>=5.3 pydicom>=2.2.0 diff --git a/setup.py b/setup.py index 619932f973..c68a41515b 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ from setuptools import setup, find_packages -VERSION = "0.16.5" +VERSION = "0.16.6" def get_version(): @@ -66,9 +66,9 @@ def get_version(): "xmltodict", "universal-analytics-python3>=1.0.1,<2", # internal packages - "fiftyone-brain>=0.8,<0.9", + "fiftyone-brain>=0.9,<0.10", "fiftyone-db>=0.3,<0.4", - "voxel51-eta>=0.7.1,<0.8", + "voxel51-eta>=0.8,<0.9", ]