[ci] move all team doc tests to civ2 (ray-project#40345)

Split doc tests into team-specific doc test jobs, so we can attribute and coverage them better There are doc tests that has team:none as owner, they will remain in civ1 Need to update serve, ml docker image to install some more dependencies; also fix a data doc test that has test assertion types of the machine type Signed-off-by: can <can@anyscale.com>
ShuN6211 · Nov 7, 2023 · a6b6898 · a6b6898
1 parent 295b7bb
commit a6b6898
Show file tree

Hide file tree

Showing 12 changed files with 148 additions and 66 deletions.
diff --git a/.buildkite/core.rayci.yml b/.buildkite/core.rayci.yml
@@ -1,6 +1,6 @@
 group: core tests
 steps:
-  # build
+  # builds
   - name: corebuild
     wanda: ci/docker/core.build.wanda.yaml
     depends_on: oss-ci-base_build
@@ -18,7 +18,7 @@ steps:
       PYTHON_VERSION: "{{matrix}}"
       EXTRA_DEPENDENCY: core
 
-  # test
+  # tests
   - label: ":ray: core: python tests"
     tags: python
     instance_type: large

diff --git a/.buildkite/data.rayci.yml b/.buildkite/data.rayci.yml
@@ -66,16 +66,45 @@ steps:
     depends_on: datanbuild
     job_env: forge
 
-  - label: ":database: data: doc gpu"
+  - label: ":database: data: doc tests"
     tags: 
       - data
-      - ci_doc
+      - doc
+    instance_type: medium
+    commands:
+      # doc tests
+      - bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... data 
+        --build-name data12build
+        --except-tags gpu
+        --only-tags doctest
+        --parallelism-per-worker 2
+      # doc examples
+      - bazel run //ci/ray_ci:test_in_docker -- //doc/... data 
+        --build-name data12build
+        --except-tags gpu,post_wheel_build,doctest
+        --parallelism-per-worker 2
+        --skip-ray-installation
+    depends_on: data12build
+    job_env: forge
+
+  - label: ":database: data: doc gpu tests"
+    tags: 
+      - data
+      - doc
       - gpu
     instance_type: gpu-large
     commands:
+      # doc tests
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //doc/... data 
+        --build-name docgpubuild
+        --only-tags doctest
+        --except-tags cpu
+      # doc examples
       - bazel run //ci/ray_ci:test_in_docker -- //doc/... data 
         --build-name docgpubuild
+        --except-tags doctest
         --only-tags gpu
+        --skip-ray-installation
     depends_on: docgpubuild
     job_env: forge
 

diff --git a/.buildkite/ml.rayci.yml b/.buildkite/ml.rayci.yml
@@ -174,6 +174,26 @@ steps:
         --only-tags minimal
     depends_on: minbuild-ml
     job_env: forge
+
+  - label: ":train: ml: doc tests"
+    tags: 
+      - train
+      - tune
+      - doc
+    instance_type: large
+    commands:
+      # doc tests
+      - bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... ml 
+        --only-tags doctest
+        --except-tags gpu
+        --parallelism-per-worker 3
+      # doc examples
+      - bazel run //ci/ray_ci:test_in_docker -- //doc/... ml 
+        --except-tags gpu,post_wheel_build,doctest,highly_parallel
+        --parallelism-per-worker 3
+        --skip-ray-installation
+    depends_on: mlbuild
+    job_env: forge
 
   - label: ":train: ml: train gpu lightning 2.0 tests"
     tags: 

diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
@@ -45,7 +45,7 @@
       --test_env=DOCKER_TLS_CERTDIR=/certs
       -- python/ray/tests/...
 
-- label: ":book: Doctest (CPU)"
+- label: ":book: civ1 doctest"
   instance_size: large
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
@@ -58,7 +58,7 @@
     - ./ci/env/install-horovod.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=doctest,-gpu,-team:core
+      --test_tag_filters=doctest,-gpu,-team:core,-team:data,-team:ml,-team:rllib,-team:serve
       python/ray/... doc/...
 
 - label: ":kubernetes: operator"

diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml
@@ -26,17 +26,6 @@
       --test_arg=--framework=torch
       rllib/...
 
-- label: ":brain: RLlib: Documentation code/examples"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
-  instance_size: medium
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_tag_filters=documentation --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
-      rllib/...
-
 # TODO(amogkam): Re-enable Ludwig tests after Ludwig supports Ray 2.0
 #- label: ":octopus: Ludwig tests and examples. Python 3.7"
 #  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]
@@ -45,7 +34,7 @@
 #    - INSTALL_LUDWIG=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
 #    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/tests/ludwig/...
 
-- label: ":book: Doc tests and examples (excluding Ray AIR examples)"
+- label: ":book: civ1 doc tests and examples"
   # Todo: check if we can modify the examples to use Ray with fewer CPUs.
   conditions:
     ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED", "RAY_CI_SERVE_AFFECTED", "RAY_CI_ML_AFFECTED"]
@@ -63,39 +52,6 @@
       --test_tag_filters=xcommit,-gpu
       doc/...
 
-- label: ":book: Doc tests and examples with time series libraries"
-  conditions:
-    ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED", "RAY_CI_SERVE_AFFECTED", "RAY_CI_ML_AFFECTED"]
-  instance_size: small
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - DOC_TESTING=1 INSTALL_TIMESERIES_LIBS=1 ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=timeseries_libs,-external,-gpu,-post_wheel_build,-doctest doc/...
-
-- label: ":book: :airplane: Ray AIR examples"
-  # Todo: check if this could be a medium test. Last time it failed because of dependency issues.
-  conditions:
-    ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED", "RAY_CI_SERVE_AFFECTED", "RAY_CI_ML_AFFECTED"]
-  instance_size: large
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - DOC_TESTING=1 ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_air,-external,-timeseries_libs,-gpu,-post_wheel_build,-doctest 
-      doc/...
-
-- label: ":book: Doc examples for external code "
-  conditions: ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED", "RAY_CI_SERVE_AFFECTED", "RAY_CI_ML_AFFECTED"]
-  instance_size: large
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - DOC_TESTING=1 ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=external,-timeseries_libs,-gpu,-post_wheel_build,-doctest 
-      doc/...
-
-
 - label: ":exploding_death_star: RLlib Contrib: A2C Tests"
   conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_RLLIB_CONTRIB_AFFECTED"]
   commands:

diff --git a/.buildkite/rllib.rayci.yml b/.buildkite/rllib.rayci.yml
@@ -160,6 +160,29 @@ steps:
     depends_on: rllibbuild
     job_env: forge
 
+  - label: ":brain: rllib: doc tests"
+    tags: 
+      - rllib_directly
+      - doc
+    instance_type: medium
+    commands:
+      # doc tests
+      - bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... rllib 
+        --except-tags gpu
+        --only-tags doctest
+        --parallelism-per-worker 2
+      # doc examples
+      - bazel run //ci/ray_ci:test_in_docker -- //doc/... rllib 
+        --except-tags gpu,post_wheel_build,timeseries_libs,doctest
+        --parallelism-per-worker 2
+        --skip-ray-installation
+      - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib 
+        --only-tags documentation
+        --parallelism-per-worker 2
+        --skip-ray-installation
+    depends_on: rllibbuild
+    job_env: forge
+
   - label: ":brain: rllib: multi-gpu tests"
     tags: 
       - rllib

diff --git a/.buildkite/serve.rayci.yml b/.buildkite/serve.rayci.yml
@@ -81,6 +81,23 @@ steps:
     depends_on: 
       - manylinux
       - servebuild
+
+  - label: ":ray-serve: serve: doc tests"
+    tags: 
+      - serve
+      - doc
+    instance_type: large
+    commands:
+      # doc tests
+      - bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... serve 
+        --only-tags doctest
+        --parallelism-per-worker 3
+      # doc examples
+      - bazel run //ci/ray_ci:test_in_docker -- //doc/... serve 
+        --except-tags gpu,post_wheel_build,timeseries_libs,doctest,xcommit
+        --parallelism-per-worker 3
+        --skip-ray-installation
+    depends_on: servebuild
     job_env: forge
 
   - label: ":ray-serve: serve: default minimal"

diff --git a/ci/docker/ml.build.Dockerfile b/ci/docker/ml.build.Dockerfile
@@ -23,9 +23,9 @@ set -euo pipefail
 
 # TODO (can): Move mosaicml to train-test-requirements.txt
 pip install "mosaicml==0.12.1"
-DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 DATA_PROCESSING_TESTING=1 INSTALL_HOROVOD=1 \
+DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 DATA_PROCESSING_TESTING=1 \
+  INSTALL_HOROVOD=1 INSTALL_TIMESERIES_LIBS=1 INSTALL_HDFS=1 \
   ./ci/env/install-dependencies.sh
-INSTALL_HDFS=1 ./ci/env/install-dependencies.sh
 
 if [[ "$RAYCI_IS_GPU_BUILD" == "true" ]]; then
   pip install -Ur ./python/requirements/ml/dl-gpu-requirements.txt

diff --git a/ci/docker/serve.build.Dockerfile b/ci/docker/serve.build.Dockerfile
@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.3-labs
+
 ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build
 FROM $DOCKER_IMAGE_BASE_BUILD
 
@@ -14,20 +16,40 @@ SHELL ["/bin/bash", "-ice"]
 
 COPY . .
 
+RUN <<EOF
+#!/bin/bash
+
+set -euo pipefail
+
 # Install custom Python version if requested.
-RUN if [[ -z $PYTHON_VERSION ]] ; then echo Not installing custom Python version ; else PYTHON=$PYTHON_VERSION ci/env/install-dependencies.sh ; fi
+if [[ -n "${PYTHON_VERSION-}" ]]; then 
+  PYTHON=$PYTHON_VERSION ci/env/install-dependencies.sh 
+else 
+  echo "Not installing custom Python version" 
+fi
 
-RUN pip install -U torch==2.0.1 torchvision==0.15.2
-RUN pip install -U tensorflow==2.13.1 tensorflow-probability==0.21.0
-RUN pip install -U --ignore-installed \
+pip install -U torch==2.0.1 torchvision==0.15.2
+pip install -U tensorflow==2.13.1 tensorflow-probability==0.21.0
+pip install -U --ignore-installed \
   -c python/requirements_compiled.txt \
   -r python/requirements.txt \
   -r python/requirements/test-requirements.txt
 
-RUN git clone https://github.com/wg/wrk.git /tmp/wrk && pushd /tmp/wrk && make -j && sudo cp wrk /usr/local/bin && popd
+# doc requirements
+#
+# TODO (shrekris-anyscale): Remove transformers after core transformer requirement 
+# is upgraded
+pip install transformers==4.30.2
+pip install -c python/requirements_compiled.txt aioboto3
 
-# Install Pydantic from source if requested.
-RUN if [[ -z $PYDANTIC_BUILD_COMMIT ]] ; \
-    then echo "Not installing Pydantic from source." ; \
-    else pip install -U git+https://github.com/pydantic/pydantic@$PYDANTIC_BUILD_COMMIT ; \
+git clone https://github.com/wg/wrk.git /tmp/wrk && pushd /tmp/wrk && make -j && sudo cp wrk /usr/local/bin && popd
+
+# Install custom Pydantic version if requested.
+if [[ -n "${PYDANTIC_BUILD_COMMIT-}" ]]; then 
+  pip install -U git+https://github.com/pydantic/pydantic@$PYDANTIC_BUILD_COMMIT
+else 
+  echo "Not installing Pydantic from source"
 fi
+
+EOF
+
diff --git a/ci/ray_ci/ml.tests.yml b/ci/ray_ci/ml.tests.yml
@@ -3,5 +3,7 @@ flaky_tests:
   - //python/ray/tune:cifar10_pytorch
   - //python/ray/tune:test_commands
   - //python/ray/train:torch_exp_tracking_mlflow
+  # doc tests
+  - //doc/source/train/examples/pytorch:convert_existing_pytorch_code_to_ray_train
   # gpu tests
   - //python/ray/train:test_torch_lightning_train 
diff --git a/doc/BUILD b/doc/BUILD
@@ -146,11 +146,24 @@ py_test_run_all_subdirectory(
         "source/serve/doc_code/http_guide/streaming_example.py",
         "source/serve/doc_code/http_guide/websockets_example.py",
         "source/serve/doc_code/vllm_example.py",
+        "source/serve/doc_code/production_guide/text_ml.py",
+        "source/serve/doc_code/fake_email_creator.py",
     ],
     extra_srcs = [],
     tags = ["exclusive", "team:serve"],
 )
 
+py_test_run_all_subdirectory(
+    size = "medium",
+    include = [
+        "source/serve/doc_code/production_guide/text_ml.py",
+        "source/serve/doc_code/fake_email_creator.py",
+    ],
+    exclude = [],
+    extra_srcs = [],
+    tags = ["exclusive", "team:serve", "xcommit"],
+)
+
 py_test_run_all_subdirectory(
     size = "medium",
     include = [

diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py
@@ -1116,7 +1116,7 @@ def randomize_block_order(
 
         Returns:
             The block-shuffled :class:`Dataset`.
-        """
+        """  # noqa: E501
 
         plan = self._plan.with_stage(RandomizeBlocksStage(seed))
 
@@ -2743,7 +2743,7 @@ def write_parquet(
                     /generated/pyarrow.parquet.write_table.html\
                         #pyarrow.parquet.write_table>`_, which is used to write out each
                 block to a file.
-        """
+        """  # noqa: E501
         datasink = _ParquetDatasink(
             path,
             arrow_parquet_args_fn=arrow_parquet_args_fn,
@@ -4721,7 +4721,7 @@ def serialize_lineage(self) -> bytes:
             .. testoutput::
 
                 Dataset(
-                   num_blocks=16,
+                   num_blocks=...,
                    num_rows=150,
                    schema={
                       sepal length (cm): double,
@@ -4804,7 +4804,7 @@ def deserialize_lineage(serialized_ds: bytes) -> "Dataset":
             .. testoutput::
 
                 Dataset(
-                   num_blocks=16,
+                   num_blocks=...,
                    num_rows=150,
                    schema={
                       sepal length (cm): double,