documentation to civ02 (ray-project#39629)

Move documentation job to civ2 - Create a doc.build wanda that install requirements-doc.txt - Compile requirements-compiled.txt together with requirements-doc.txt - DocBuildContainer simply runs 'make html' Signed-off-by: can <can@anyscale.com>
ShuN6211 · Sep 22, 2023 · d3eb64b · d3eb64b
1 parent af12c2f
commit d3eb64b
Show file tree

Hide file tree

Showing 10 changed files with 103 additions and 24 deletions.
diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml
@@ -24,6 +24,10 @@ steps:
     wanda: ci/docker/base.ml.wanda.yaml
     depends_on: oss-ci-base_test
 
+  - name: docbuild
+    wanda: ci/docker/doc.build.wanda.yaml
+    depends_on: oss-ci-base_build
+
   - name: corebuild
     wanda: ci/docker/core.build.wanda.yaml
     depends_on: oss-ci-base_build

diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml
@@ -24,6 +24,13 @@ steps:
     depends_on: manylinux
     job_env: manylinux
 
+  - label: ":tapioca: build: doc"
+    instance_type: medium
+    commands:
+      - bazel run //ci/ray_ci:build_in_docker -- doc 
+    depends_on: docbuild
+    job_env: docbuild
+
   - label: ":tapioca: build: ray py38 cu118 docker"
     instance_type: medium
     commands:

diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
@@ -420,26 +420,6 @@
     - echo "--- Running the script with fault of networking delay"
     - ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/streaming_llm.yaml --working-dir python/ray/tests/chaos -- python streaming_llm.py --num_queries_per_task=100 --num_tasks=2 --num_words_per_query=100
 
-
-- label: ":book: Documentation"
-  commands:
-    - export LINT=1
-    - ./ci/env/install-dependencies.sh
-    # Specifying above somehow messes up the Ray install.
-    # Uninstall and re-install Ray so that we can use Ray Client
-    # (remove thirdparty_files to sidestep an issue with psutil).
-    - pip uninstall -y ray && rm -rf /ray/python/ray/thirdparty_files
-    - pushd /ray && git clean -f -f -x -d -e .whl -e python/ray/dashboard/client && popd
-    - bazel clean --expunge
-    - export WANDB_MODE=offline
-    # Horovod needs to be installed separately (needed for API ref imports)
-    - ./ci/env/install-horovod.sh
-    # See https://stackoverflow.com/questions/63383400/error-cannot-uninstall-ruamel-yaml-while-creating-docker-image-for-azure-ml-a
-    # Pin urllib to avoid downstream ssl incompatibility issues. This matches requirements-doc.txt.
-    - pip install "mosaicml==0.12.1" "urllib3<1.27" "typing-extensions==4.5.0" --ignore-installed
-    - ./ci/env/env_info.sh
-    - ./ci/ci.sh build_sphinx_docs
-
 - label: ":octopus: Tune multinode tests"
   conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]
   instance_size: medium

diff --git a/ci/ci.sh b/ci/ci.sh
@@ -128,7 +128,8 @@ compile_pip_dependencies() {
     "${WORKSPACE_DIR}/python/requirements/ml/train-requirements.txt" \
     "${WORKSPACE_DIR}/python/requirements/ml/train-test-requirements.txt" \
     "${WORKSPACE_DIR}/python/requirements/ml/tune-requirements.txt" \
-    "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt"
+    "${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt" \
+    "${WORKSPACE_DIR}/doc/requirements-doc.txt"
 
   # Remove some pins from upstream dependencies:
   # ray, xgboost-ray, lightgbm-ray, tune-sklearn

diff --git a/ci/docker/doc.build.Dockerfile b/ci/docker/doc.build.Dockerfile
@@ -0,0 +1,22 @@
+ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build
+FROM $DOCKER_IMAGE_BASE_BUILD
+
+# Unset dind settings; we are using the host's docker daemon.
+ENV DOCKER_TLS_CERTDIR=
+ENV DOCKER_HOST=
+ENV DOCKER_TLS_VERIFY=
+ENV DOCKER_CERT_PATH=
+
+SHELL ["/bin/bash", "-ice"]
+
+COPY . .
+
+RUN pip install -U --ignore-installed  \
+  -c python/requirements_compiled.txt \
+  -r python/requirements.txt \
+  -r python/requirements/test-requirements.txt \
+  -r python/requirements/lint-requirements.txt \
+  -r doc/requirements-doc.txt 
+
+RUN HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_MXNET=1 \
+  pip install -U --ignore-installed  -c python/requirements_compiled.txt horovod
diff --git a/ci/docker/doc.build.wanda.yaml b/ci/docker/doc.build.wanda.yaml
@@ -0,0 +1,11 @@
+name: "docbuild"
+froms: ["cr.ray.io/rayproject/oss-ci-base_build"]
+dockerfile: ci/docker/doc.build.Dockerfile
+srcs:
+  - python/requirements.txt
+  - python/requirements_compiled.txt
+  - python/requirements/test-requirements.txt
+  - python/requirements/lint-requirements.txt
+  - doc/requirements-doc.txt
+tags:
+  - cr.ray.io/rayproject/docbuild
diff --git a/ci/ray_ci/builder.py b/ci/ray_ci/builder.py
@@ -1,6 +1,7 @@
 import click
 
 from ci.ray_ci.builder_container import PYTHON_VERSIONS, BuilderContainer
+from ci.ray_ci.doc_builder_container import DocBuilderContainer
 from ci.ray_ci.forge_container import ForgeContainer
 from ci.ray_ci.docker_container import PLATFORM, DockerContainer
 from ci.ray_ci.container import _DOCKER_ECR_REPO
@@ -11,7 +12,7 @@
 @click.argument(
     "artifact_type",
     required=True,
-    type=click.Choice(["wheel", "docker"]),
+    type=click.Choice(["wheel", "doc", "docker"]),
 )
 @click.option(
     "--python-version",
@@ -50,20 +51,32 @@ def main(
         build_docker(python_version, platform, image_type)
         return
 
+    if artifact_type == "doc":
+        logger.info("Building ray docs")
+        build_doc()
+        return
+
     raise ValueError(f"Invalid artifact type {artifact_type}")
 
 
 def build_wheel(python_version: str) -> None:
     """
-    Build a wheel artifact
+    Build a wheel artifact.
     """
     BuilderContainer(python_version).run()
     ForgeContainer().upload_wheel()
 
 
 def build_docker(python_version: str, platform: str, image_type: str) -> None:
     """
-    Build a container artifact
+    Build a container artifact.
     """
     BuilderContainer(python_version).run()
     DockerContainer(python_version, platform, image_type).run()
+
+
+def build_doc() -> None:
+    """
+    Build a doc artifact.
+    """
+    DocBuilderContainer().run()
diff --git a/ci/ray_ci/doc_builder_container.py b/ci/ray_ci/doc_builder_container.py
@@ -0,0 +1,15 @@
+from ci.ray_ci.container import Container
+
+
+class DocBuilderContainer(Container):
+    def __init__(self) -> None:
+        super().__init__("docbuild")
+        self.install_ray()
+
+    def run(self) -> None:
+        self.run_script(
+            [
+                "cd doc",
+                "FAST=True make html",
+            ]
+        )
diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
@@ -37,3 +37,7 @@ urllib3 < 1.27
 # For the API docs
 fastapi==0.99.1
 grpcio==1.57.0
+
+# Build dependencies
+mosaicml
+typing-extensions
diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt
@@ -28,6 +28,7 @@ applicationinsights==0.11.10
 argcomplete==1.12.3
 argon2-cffi==23.1.0
 argon2-cffi-bindings==21.2.0
+argparse==1.4.0
 array-record==0.4.0
 arrow==1.2.3
 asttokens==2.4.0
@@ -37,6 +38,7 @@ async-generator==1.10
 async-timeout==4.0.3
 asyncmock==0.4.2
 attrs==21.4.0
+autodoc-pydantic==1.6.1
 autograd==1.6.2
 autorom==0.6.1 ; platform_machine != "arm64"
 autorom-accept-rom-license==0.6.1
@@ -117,6 +119,7 @@ dm-tree==0.1.8
 dnspython==2.4.2
 docker==6.1.3
 docker-pycreds==0.4.0
+docstring-parser==0.15
 docutils==0.17.1
 dopamine-rl==4.0.5 ; (sys_platform != "darwin" or platform_machine != "arm64") and python_version >= "3.8"
 dragonfly-opt==0.1.7
@@ -194,6 +197,7 @@ h5py==3.7.0
 hebo @ git+https://github.com/huawei-noah/HEBO@9a2a674c22518eed35a8b98e5134576741a95410#subdirectory=HEBO
 higher==0.2.1
 hjson==3.1.0
+horovod==0.28.1
 hpbandster==0.7.4
 httpcore==0.15.0
 httplib2==0.20.4
@@ -281,6 +285,7 @@ mock==5.1.0
 modin==0.22.2 ; python_version >= "3.8"
 monotonic==1.6
 more-itertools==10.1.0
+mosaicml==0.3.1
 moto==4.0.7
 mpmath==1.3.0
 msal==1.18.0b1
@@ -384,6 +389,7 @@ pycodestyle==2.7.0
 pycparser==2.21
 pycryptodome==3.18.0
 pydantic==1.10.12
+pydata-sphinx-theme==0.8.1
 pydeprecate==0.3.2
 pydot==1.4.2
 pydub==0.25.1
@@ -430,6 +436,7 @@ python-lsp-jsonrpc==1.0.0
 python-multipart==0.0.6
 python-snappy==0.6.1
 pytorch-lightning==1.6.5
+pytorch-ranger==0.1.1
 pytz==2022.7.1
 pyu2f==0.1.5
 pyvmomi==8.0.1.0.2
@@ -481,13 +488,25 @@ snowballstemmer==2.2.0
 sortedcontainers==2.4.0
 soupsieve==2.5
 sphinx==4.3.2
+sphinx-book-theme==0.3.3
+sphinx-click==3.0.2
+sphinx-copybutton==0.4.0
+sphinx-design==0.4.1
+sphinx-external-toc==0.2.4
+sphinx-jsonschema==1.17.2
+sphinx-remove-toctrees==0.0.3
+sphinx-sitemap==2.2.0
+sphinx-tabs==3.4.0
 sphinx-togglebutton==0.2.3
+sphinx-version-warning==1.1.2
 sphinxcontrib-applehelp==1.0.4
 sphinxcontrib-devhelp==1.0.2
 sphinxcontrib-htmlhelp==2.0.1
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-redoc==1.6.0
 sphinxcontrib-serializinghtml==1.1.5
+sphinxemoji==0.2.0
 sqlalchemy==1.4.17
 sqlparse==0.4.4
 sshpubkeys==3.3.1
@@ -528,6 +547,7 @@ toolz==0.12.0
 torch==2.0.1 ; python_version > "3.7"
 torch-cluster==1.6.1
 torch-geometric==2.3.1
+torch-optimizer==0.1.0
 torch-scatter==2.1.1
 torch-sparse==0.6.17
 torch-spline-conv==1.2.2
@@ -540,6 +560,7 @@ tqdm==4.64.1
 traitlets==5.9.0
 transformers==4.19.1
 trustme==0.9.0
+tune-sklearn @ git+https://github.com/ray-project/tune-sklearn@master
 typeguard==2.13.3
 typer==0.9.0
 types-pyyaml==6.0.12.2
@@ -571,6 +592,7 @@ xlrd==2.0.1
 xmltodict==0.13.0
 xxhash==3.3.0
 y-py==0.6.0
+yahp==0.1.1
 yarl==1.9.2
 ypy-websocket==0.8.4
 yq==3.2.2