diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 3819447db96b..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: 4.5.0.99.{build} - -image: Visual Studio 2015 -platform: x64 -configuration: - - '3.8' - -# only build on 'master' and pull requests targeting it -branches: - only: - - master - -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python - -clone_depth: 5 - -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - ps: | - $env:ALLOW_SKIP_ARROW_TESTS = "1" - $env:APPVEYOR = "true" - $env:CMAKE_BUILD_PARALLEL_LEVEL = 4 - $env:MINICONDA = "C:\Miniconda3-x64" - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - -build: false - -test_script: - - conda config --remove channels defaults - - conda config --add channels nodefaults - - conda config --add channels conda-forge - - conda config --set channel_priority strict - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test-windows.ps1 diff --git a/.ci/download-r-revdeps.R b/.ci/download-r-revdeps.R new file mode 100644 index 000000000000..f6e71def70ff --- /dev/null +++ b/.ci/download-r-revdeps.R @@ -0,0 +1,75 @@ +loadNamespace("crandep") + +PKG_DIR <- commandArgs(trailing = TRUE)[[1L]] + +.log <- function(msg) { + cat(sprintf("[download-revdeps] %s\n", msg)) +} + +# get all of lightgbm's reverse dependencies +depDF <- crandep::get_dep( + name = "lightgbm" + , type = "all" + , reverse = TRUE +) +reverse_deps <- depDF[["to"]] +.log(sprintf("found %i reverse deps", length(reverse_deps))) + +# skip some dependencies with known issues: +# +# * 'misspi' (https://github.com/microsoft/LightGBM/issues/6741) +# * 'qeML' (checks take 45+ minutes to run) +deps_to_skip <- c( + "misspi" + , "qeML" +) +.log(sprintf("excluding %i reverse deps: %s", length(deps_to_skip), toString(deps_to_skip))) +reverse_deps <- reverse_deps[!reverse_deps %in% deps_to_skip] + +.log(sprintf("checking the following packages: %s", toString(reverse_deps))) + +# get the superset of all their dependencies +# (all of the packages needed to run 'R CMD check' on lightgbm's reverse deps) +their_deps <- unlist( + unname( + sapply( + X = reverse_deps + , FUN = function(pkg) { + return( + crandep::get_dep( + name = pkg + , type = "all" + , reverse = FALSE + )[["to"]] + ) + } + ) + ) +) + +all_deps <- sort(unique(c(their_deps, reverse_deps))) + +# don't try to install 'lightgbm', or packages that ship with the R standard library +all_deps <- all_deps[!all_deps %in% c("grid", "methods", "lightgbm", "parallel", "stats", "utils")] + +.log(sprintf("packages required to run these checks: %i", length(all_deps))) + +.log("installing all packages required to check reverse dependencies") + +# install only the strong dependencies of all those packages +install.packages( # nolint: undesirable_function + pkgs = all_deps + , repos = "https://cran.r-project.org" + , dependencies = c("Depends", "Imports", "LinkingTo") + , type = "both" + , Ncpus = parallel::detectCores() +) + +# get source tarballs, to be checked with 'R CMD check' +print(sprintf("--- downloading reverse dependencies to check (%i)", length(reverse_deps))) +download.packages( + pkgs = reverse_deps + , destdir = PKG_DIR + , repos = "https://cran.r-project.org" + , type = "source" +) diff --git a/.ci/run-r-revdepchecks.R b/.ci/run-r-revdepchecks.R new file mode 100644 index 000000000000..5ea0323229bd --- /dev/null +++ b/.ci/run-r-revdepchecks.R @@ -0,0 +1,30 @@ +options(repos = "https://cran.r-project.org") + +check_dir <- commandArgs(trailing = TRUE)[[1L]] + +tools::check_packages_in_dir( + dir = check_dir + , check_args = c("--no-manual", "--run-dontrun", "--run-donttest") + , clean = TRUE + , all = TRUE + # only check one package at a time, to avoid oversubscribing CPUs + , Ncpus = 2L + # only test the libraries found in `check_dir` + , reverse = FALSE +) + +all_checks_passed <- tools::summarize_check_packages_in_dir_results( + dir = check_dir + , all = TRUE +) + +if (!isTRUE(all_checks_passed)) { + invisible( + tools::summarize_check_packages_in_dir_results( + dir = check_dir + , all = TRUE + , full = TRUE + ) + ) + stop("Some checks failed! See results above.") +} diff --git a/.ci/run-revdep-checks.sh b/.ci/run-revdep-checks.sh new file mode 100755 index 000000000000..92cc0ab60f56 --- /dev/null +++ b/.ci/run-revdep-checks.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e -u -o pipefail + +CHECKS_OUTPUT_DIR=/tmp/lgb-revdepchecks +mkdir -p "${CHECKS_OUTPUT_DIR}" + +# Pre-install all of lightgbm's reverse dependencies, and all of their dependencies, +# preferring precompiled binaries where available. +# +# This is done for speed... tools::check_packages_in_dir() only performs source +# installs of all packages, which results in lots of compilation. +# +# ref: https://github.com/wch/r-source/blob/594b842678e932088b16ec0cd3c39714a141eed9/src/library/tools/R/checktools.R#L295 +# +# {lightgbm} checks do not need to care about that... as of this writing, nothing has {lightgbm} as a +# 'LinkingTo' dependency or otherwise needs {lightgbm} at build time. +Rscript ./.ci/download-r-revdeps.R "${CHECKS_OUTPUT_DIR}" + +# build and install 'lightgbm' +sh ./build-cran-package.sh --no-build-vignettes +R CMD INSTALL --with-keep.source ./lightgbm_*.tar.gz + +# run 'R CMD check' on lightgbm's reverse dependencies +Rscript ./.ci/run-r-revdepchecks.R "${CHECKS_OUTPUT_DIR}" + +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/EIX_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/SHAPforxgboost_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/cbl_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/bonsai_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/fastshap_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/fastml_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/fastshap_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/predhy.GUI_*.tar.gz || true +# qeML vignettes take a very very long time to run +#R CMD check --no-manual --run-dontrun --run-donttest --ignore-vignettes ${CHECKS_OUTPUT_DIR}/qeML_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/mllrns_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/predhy_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/r2pmml_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/stackgbm_*.tar.gz || true +# R CMD check --no-manual --run-dontrun --run-donttest ${CHECKS_OUTPUT_DIR}/vip_*.tar.gz || true diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index f6bb354bc5a9..000000000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,136 +0,0 @@ -name: CUDA Version - -on: - push: - branches: - - master - pull_request: - branches: - - master - # Run manually by clicking a button in the UI - workflow_dispatch: - inputs: - restart_docker: - description: 'Restart nvidia-docker on the runner before building?' - required: true - type: boolean - default: false - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # Optionally reinstall + restart docker on the runner before building. - # This is safe as long as only 1 of these jobs runs at a time. - restart-docker: - name: set up docker - runs-on: [self-hosted, linux] - timeout-minutes: 30 - steps: - - name: Setup or update software on host machine - if: ${{ inputs.restart_docker }} - run: | - # install core packages - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - lsb-release \ - software-properties-common - # set up nvidia-docker - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: mark job successful - run: | - exit 0 - test: - name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - needs: [restart-docker] - container: - image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }} - env: - CMAKE_BUILD_PARALLEL_LEVEL: 4 - COMPILER: ${{ matrix.compiler }} - CONDA: /tmp/miniforge - DEBIAN_FRONTEND: noninteractive - METHOD: ${{ matrix.method }} - OS_NAME: linux - PYTHON_VERSION: ${{ matrix.python_version }} - TASK: ${{ matrix.task }} - SKBUILD_STRICT_CONFIG: true - options: --gpus all - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - method: wheel - compiler: gcc - python_version: "3.10" - cuda_version: "12.6.1" - linux_version: "ubuntu22.04" - task: cuda - - method: source - compiler: gcc - python_version: "3.12" - cuda_version: "12.2.2" - linux_version: "ubuntu22.04" - task: cuda - - method: pip - compiler: clang - python_version: "3.11" - cuda_version: "11.8.0" - linux_version: "ubuntu20.04" - task: cuda - steps: - - name: Install latest git and sudo - run: | - apt-get update - apt-get install --no-install-recommends -y \ - ca-certificates \ - software-properties-common - add-apt-repository ppa:git-core/ppa -y - apt-get update - apt-get install --no-install-recommends -y \ - git \ - sudo - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export PATH=$CONDA/bin:$PATH - - # check GPU usage - nvidia-smi - - # build and test - $GITHUB_WORKSPACE/.ci/setup.sh - $GITHUB_WORKSPACE/.ci/test.sh - all-cuda-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml deleted file mode 100644 index acc40026f9b8..000000000000 --- a/.github/workflows/python_package.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: Python-package - -on: - push: - branches: - - master - pull_request: - branches: - - master - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - CMAKE_BUILD_PARALLEL_LEVEL: 4 - SKBUILD_STRICT_CONFIG: true - -jobs: - test: - name: ${{ matrix.task }} ${{ matrix.method }} (${{ matrix.os }}, Python ${{ matrix.python_version }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - os: macos-13 - task: regular - python_version: '3.10' - - os: macos-13 - task: sdist - python_version: '3.11' - - os: macos-13 - task: bdist - python_version: '3.8' - - os: macos-13 - task: if-else - python_version: '3.9' - - os: macos-14 - task: bdist - method: wheel - python_version: '3.10' - - os: macos-13 - task: mpi - method: source - python_version: '3.11' - - os: macos-13 - task: mpi - method: pip - python_version: '3.12' - - os: macos-13 - task: mpi - method: wheel - python_version: '3.9' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - shell: bash - run: | - export TASK="${{ matrix.task }}" - export METHOD="${{ matrix.method }}" - export PYTHON_VERSION="${{ matrix.python_version }}" - if [[ "${{ matrix.os }}" == "macos-14" ]]; then - # use clang when creating macOS release artifacts - export COMPILER="clang" - export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "macos-13" ]]; then - export COMPILER="gcc" - export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - export COMPILER="clang" - export OS_NAME="linux" - fi - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export CONDA=${HOME}/miniforge - export PATH=${CONDA}/bin:${PATH} - $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 - $GITHUB_WORKSPACE/.ci/test.sh || exit 1 - - name: upload wheels - if: ${{ matrix.method == 'wheel' && matrix.os == 'macos-14' }} - uses: actions/upload-artifact@v4 - with: - name: macosx-arm64-wheel - path: dist/*.whl - test-latest-versions: - name: Python - latest versions (ubuntu-latest) - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Create wheel - run: | - docker run \ - --rm \ - --env CMAKE_BUILD_PARALLEL_LEVEL=${{ env.CMAKE_BUILD_PARALLEL_LEVEL }} \ - -v $(pwd):/opt/lgb-build \ - -w /opt/lgb-build \ - lightgbm/vsts-agent:manylinux_2_28_x86_64 \ - /bin/bash -c 'PATH=/opt/miniforge/bin:$PATH sh ./build-python.sh bdist_wheel --nomp' - - name: Test compatibility - run: | - docker run \ - --rm \ - -v $(pwd):/opt/lgb-build \ - -w /opt/lgb-build \ - python:3.12 \ - /bin/bash ./.ci/test-python-latest.sh - test-oldest-versions: - name: Python - oldest supported versions (ubuntu-latest) - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Create wheel - run: | - docker run \ - --rm \ - --env CMAKE_BUILD_PARALLEL_LEVEL=${{ env.CMAKE_BUILD_PARALLEL_LEVEL }} \ - -v $(pwd):/opt/lgb-build \ - -w /opt/lgb-build \ - lightgbm/vsts-agent:manylinux_2_28_x86_64 \ - /bin/bash -c 'PATH=/opt/miniforge/bin:$PATH sh ./build-python.sh bdist_wheel --nomp' - - name: Test compatibility - run: | - docker run \ - --rm \ - -v $(pwd):/opt/lgb-build \ - -w /opt/lgb-build \ - python:3.7 \ - /bin/bash ./.ci/test-python-oldest.sh - all-python-package-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test, test-latest-versions, test-oldest-versions] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml deleted file mode 100644 index 66e05a18ba1f..000000000000 --- a/.github/workflows/r_package.yml +++ /dev/null @@ -1,350 +0,0 @@ -name: R-package - -on: - push: - branches: - - master - pull_request: - branches: - - master - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # in CMake-driven builds, parallelize compilation - CMAKE_BUILD_PARALLEL_LEVEL: 4 - # on Debian-based images, avoid interactive prompts - DEBIAN_FRONTEND: noninteractive - # parallelize compilation (extra important for Linux, where CRAN doesn't supply pre-compiled binaries) - MAKEFLAGS: "-j4" - # hack to get around this: - # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html - _R_CHECK_SYSTEM_CLOCK_: 0 - # ignore R CMD CHECK NOTE checking how long it has - # been since the last submission - _R_CHECK_CRAN_INCOMING_REMOTE_: 0 - # CRAN ignores the "installed size is too large" NOTE, - # so our CI can too. Setting to a large value here just - # to catch extreme problems - _R_CHECK_PKG_SIZES_THRESHOLD_: 100 - -jobs: - test: - name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) - runs-on: ${{ matrix.os }} - container: ${{ matrix.container }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - ################ - # CMake builds # - ################ - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: ubuntu-latest - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: null - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cmake - container: null - # Visual Studio 2019 - - os: windows-2019 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 3.6 - build_type: cmake - container: null - # Visual Studio 2022 - - os: windows-2022 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 4.3 - build_type: cmake - container: null - ############### - # CRAN builds # - ############### - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cran - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cran - container: null - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cran - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null - # macos-14 = arm64 - - os: macos-14 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null - steps: - - name: Prevent conversion of line endings on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh - run: git config --global core.autocrlf false - - name: Install packages used by third-party actions - if: startsWith(matrix.os, 'ubuntu') - shell: bash - run: | - apt-get update -y - apt-get install --no-install-recommends -y \ - ca-certificates \ - dirmngr \ - gpg \ - gpg-agent \ - software-properties-common \ - sudo - # install newest version of git - # ref: - # - https://unix.stackexchange.com/a/170831/550004 - # - https://git-scm.com/download/linux - add-apt-repository ppa:git-core/ppa -y - apt-get update -y - apt-get install --no-install-recommends -y \ - git - - name: Trust git cloning LightGBM - if: startsWith(matrix.os, 'ubuntu') - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2 - - name: Install tinytex - if: startsWith(matrix.os, 'windows') - uses: r-lib/actions/setup-tinytex@v2 - env: - CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex - TINYTEX_INSTALLER: TinyTeX - - name: Setup and run tests on Linux and macOS - if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') - shell: bash - run: | - export TASK="${{ matrix.task }}" - export COMPILER="${{ matrix.compiler }}" - if [[ "${{ matrix.os }}" =~ ^macos ]]; then - export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - export OS_NAME="linux" - export IN_UBUNTU_BASE_CONTAINER="true" - fi - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export R_VERSION="${{ matrix.r_version }}" - export R_BUILD_TYPE="${{ matrix.build_type }}" - $GITHUB_WORKSPACE/.ci/setup.sh - $GITHUB_WORKSPACE/.ci/test.sh - - name: Setup and run tests on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh -command ". {0}" - run: | - $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE - $env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-') - $env:TOOLCHAIN = "${{ matrix.toolchain }}" - $env:R_VERSION = "${{ matrix.r_version }}" - $env:R_BUILD_TYPE = "${{ matrix.build_type }}" - $env:COMPILER = "${{ matrix.compiler }}" - $env:TASK = "${{ matrix.task }}" - & "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1" - test-r-sanitizers: - name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: wch1/r-debug - strategy: - fail-fast: false - matrix: - include: - - r_customization: san - compiler: gcc - - r_customization: csan - compiler: clang - steps: - - name: Trust git cloning LightGBM - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages - shell: bash - run: | - RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} - RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 - - name: Run tests with sanitizers - shell: bash - run: | - cd R-package/tests - exit_code=0 - RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 - cat ./tests.log - exit ${exit_code} - test-r-extra-checks: - name: r-package (${{ matrix.image }}, R-devel) - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - # references: - # * CRAN "additional checks": https://cran.r-project.org/web/checks/check_issue_kinds.html - # * images: https://r-hub.github.io/containers/containers.html - image: - - clang16 - - clang17 - - clang18 - - clang19 - - gcc14 - - intel - - rchk - runs-on: ubuntu-latest - container: ghcr.io/r-hub/containers/${{ matrix.image }}:latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2 - - name: Install LaTeX - shell: bash - run: | - if type -f apt 2>&1 > /dev/null; then - apt-get update - apt-get install --no-install-recommends -y \ - devscripts \ - texinfo \ - texlive-latex-extra \ - texlive-latex-recommended \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - tidy \ - qpdf - else - yum update -y - yum install -y \ - devscripts \ - qpdf \ - texinfo \ - texinfo-tex \ - texlive-latex \ - tidy - fi - - name: Install packages and run tests - shell: bash - run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - - # 'rchk' isn't run through 'R CMD check', use the approach documented at - # https://r-hub.github.io/containers/local.html - if [[ "${{ matrix.image }}" =~ "rchk" ]]; then - r-check "$(pwd)" \ - | tee ./rchk-logs.txt 2>&1 - - # the '-v' exceptions below are from R/rchk itself and not LightGBM: - # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 - if grep -E '\[PB\]|ERROR' ./rchk-logs.txt \ - | grep -v 'too many states' \ - > /dev/null; \ - then - echo "rchk found issues" - exit 1 - else - echo "rchk did not find any issues" - exit 0 - fi - fi - - # 'testthat' is not needed by 'rchk', so avoid installing it until here - Rscript -e "install.packages('testthat', repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - - if [[ "${{ matrix.image }}" =~ "clang" ]]; then - # allowing the following NOTEs (produced by default in the clang images): - # - # * checking compilation flags used ... NOTE - # Compilation used the following non-portable flag(s): - # ‘-Wp,-D_FORTIFY_SOURCE=3’ - # - # even though CRAN itself sets that: - # https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang - # - declare -i allowed_notes=1 - else - declare -i allowed_notes=0 - fi - - bash .ci/run-r-cmd-check.sh \ - "$(echo lightgbm_$(head -1 VERSION.txt).tar.gz)" \ - "${allowed_notes}" - all-r-package-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-extra-checks] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/r_revdepchecks.yml b/.github/workflows/r_revdepchecks.yml new file mode 100644 index 000000000000..3b68dd9255fd --- /dev/null +++ b/.github/workflows/r_revdepchecks.yml @@ -0,0 +1,87 @@ +name: R reverse dependency checks + +on: + push: + # Run manually by clicking a button in the UI + workflow_dispatch: + # Run on the 5th day of every month + schedule: + - cron: '0 0 5 * *' + +# automatically cancel in-progress builds if another commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # parallelize compilation (extra important for Linux, where CRAN doesn't supply pre-compiled binaries) + MAKEFLAGS: "-j4" + # ignore R CMD CHECK NOTE checking how long it has + # been since the last submission + _R_CHECK_CRAN_INCOMING_REMOTE_: 0 + # CRAN ignores the "installed size is too large" NOTE, + # so our CI can too. Setting to a large value here just + # to catch extreme problems + _R_CHECK_PKG_SIZES_THRESHOLD_: 100 + # ensure that the locally-installed version of 'tidy' is used + # ref: https://cran.r-project.org/doc/manuals/R-exts.html#Checking-packages + R_TIDYCMD: '/usr/local/bin/tidy' + +jobs: + test: + name: r-revdepchecks (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + # NOTE: platforms that CRAN provides pre-compiled binaries on (like macOS and Windows) + # are preferable for this, because packages installs are much faster + matrix: + os: + - macos-14 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - uses: r-lib/actions/setup-pandoc@v2 + - uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + - name: install system dependencies + run: | + brew install \ + automake \ + basictex \ + checkbashisms \ + libomp \ + qpdf + echo "/Library/TeX/texbin/" >> ${GITHUB_PATH} + sudo /Library/TeX/texbin/tlmgr --verify-repo=none update --self + sudo /Library/TeX/texbin/tlmgr --verify-repo=none install inconsolata helvetic rsfs + - name: install tidy + run: | + # install tidy v5.8.0 + # ref: https://groups.google.com/g/r-sig-mac/c/7u_ivEj4zhM + TIDY_URL=https://github.com/htacg/tidy-html5/releases/download/5.8.0/tidy-5.8.0-macos-x86_64+arm64.pkg + curl -sL ${TIDY_URL} -o tidy.pkg + sudo installer \ + -pkg "$(pwd)/tidy.pkg" \ + -target / + - name: build package + run: | + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores(), type = 'binary')" + sh ./build-cran-package.sh + - name: run revdepchecks + run: | + Rscript -e "install.packages(c('crandep'), repos = 'https://cran.r-project.org', type = 'binary')" + bash ./.ci/run-revdep-checks.sh + all-r-revdepchecks-jobs-successful: + if: always() + runs-on: ubuntu-latest + needs: [test] + steps: + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.gitignore b/.gitignore index efa59fdfc962..4d77a2d8abef 100644 --- a/.gitignore +++ b/.gitignore @@ -405,6 +405,7 @@ python-package/lightgbm/VERSION.txt # R build artefacts **/autom4te.cache/ +*.Rcheck/ R-package/conftest* R-package/config.status !R-package/data/agaricus.test.rda @@ -438,7 +439,6 @@ tests/distributed/train* tests/distributed/model* tests/distributed/predict* - # Files from interactive R sessions .Rproj.user **/.Rapp.history diff --git a/.vsts-ci.yml b/.vsts-ci.yml deleted file mode 100644 index 3a111e10898e..000000000000 --- a/.vsts-ci.yml +++ /dev/null @@ -1,481 +0,0 @@ -trigger: - branches: - include: - - master - tags: - include: - - v* -pr: -- master -variables: - AZURE: 'true' - CMAKE_BUILD_PARALLEL_LEVEL: 4 - PYTHON_VERSION: '3.12' - runCodesignValidationInjection: false - skipComponentGovernanceDetection: true - Codeql.Enabled: false - Codeql.SkipTaskAutoInjection: true - DOTNET_CLI_TELEMETRY_OPTOUT: true - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - SKBUILD_STRICT_CONFIG: true -resources: - # The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because - # LightGBM's CI jobs write files in the source directory. - # - # For all the containers included here, all other directories that Azure mounts in are mounted as read-only - # to minimize the risk of side effects from one run affecting future runs. - # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container - containers: - - container: linux-artifact-builder - image: lightgbm/vsts-agent:manylinux_2_28_x86_64 - mountReadOnly: - work: false - externals: true - tools: true - tasks: true - - container: ubuntu-latest - image: 'ubuntu:22.04' - options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - mountReadOnly: - work: false - externals: true - tools: true - tasks: true - - container: rbase - image: wch1/r-debug - mountReadOnly: - work: false - externals: true - tools: true - tasks: true -jobs: -########################################### -- job: Maintenance -########################################### - pool: mariner-20240410-0 - container: ubuntu-latest - # routine maintenance (like periodically deleting old files), - # to be run on 1 random CI runner in the self-hosted pool each runner - steps: - - script: | - print-diagnostics(){ - echo "---- df -h -m ----" - df -h -m - echo "---- docker system df ----" - /tmp/docker system df - echo "---- docker images ----" - /tmp/docker images - } - # check disk usage - print-diagnostics - # remove old containers, container images, volumes - # ref: https://stackoverflow.com/a/32723127/3986677) - echo "---- running 'docker system prune' ----" - /tmp/docker system prune \ - --all \ - --force \ - --filter until=720h - # check disk usage again - print-diagnostics - displayName: clean -########################################### -- job: Linux -########################################### - variables: - COMPILER: gcc - SETUP_CONDA: 'false' - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: mariner-20240410-0 - container: linux-artifact-builder - strategy: - matrix: - regular: - TASK: regular - PYTHON_VERSION: '3.10' - sdist: - TASK: sdist - PYTHON_VERSION: '3.8' - bdist: - TASK: bdist - PYTHON_VERSION: '3.9' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - PYTHON_VERSION: '3.9' - gpu_source: - TASK: gpu - METHOD: source - swig: - TASK: swig - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - - script: | - git clean -d -f -x - displayName: 'Clean source directory' - - script: | - echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' - displayName: 'Add commit hash to artifacts archive' - - task: Bash@3 - displayName: Setup - inputs: - filePath: $(Build.SourcesDirectory)/.ci/setup.sh - targetType: filePath - - task: Bash@3 - displayName: Test - inputs: - filePath: $(Build.SourcesDirectory)/.ci/test.sh - targetType: filePath - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_latest -########################################### - variables: - COMPILER: clang-17 - DEBIAN_FRONTEND: 'noninteractive' - IN_UBUNTU_BASE_CONTAINER: 'true' - OS_NAME: 'linux' - SETUP_CONDA: 'true' - pool: mariner-20240410-0 - container: ubuntu-latest - strategy: - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - bdist: - TASK: bdist - PYTHON_VERSION: '3.10' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - mpi_pip: - TASK: mpi - METHOD: pip - PYTHON_VERSION: '3.11' - mpi_wheel: - TASK: mpi - METHOD: wheel - PYTHON_VERSION: '3.9' - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: '3.11' - gpu_pip: - TASK: gpu - METHOD: pip - PYTHON_VERSION: '3.10' - gpu_wheel: - TASK: gpu - METHOD: wheel - PYTHON_VERSION: '3.9' - cpp_tests: - TASK: cpp-tests - METHOD: with-sanitizers - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - CONDA=$HOME/miniforge - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install sudo' - - script: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends git - git clean -d -f -x - displayName: 'Clean source directory' - - task: Bash@3 - displayName: Setup - inputs: - filePath: $(Build.SourcesDirectory)/.ci/setup.sh - targetType: 'filePath' - - task: Bash@3 - displayName: Test - inputs: - filePath: $(Build.SourcesDirectory)/.ci/test.sh - targetType: 'filePath' -########################################### -- job: QEMU_multiarch -########################################### - variables: - BUILD_DIRECTORY: /LightGBM - COMPILER: gcc - PRODUCES_ARTIFACTS: 'true' - pool: - vmImage: ubuntu-22.04 - timeoutInMinutes: 180 - strategy: - matrix: - bdist: - TASK: bdist - ARCH: aarch64 - steps: - - script: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - binfmt-support \ - qemu \ - qemu-user \ - qemu-user-static - displayName: 'Install QEMU' - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: 'Enable Docker multi-architecture support' - - script: | - git clean -d -f -x - displayName: 'Clean source directory' - - script: | - cat > docker-script.sh <