Add shell.nix (qdrant#4821)

AKJUS · Aug 9, 2024 · f6493e7 · f6493e7
1 parent 8030504
commit f6493e7
Show file tree

Hide file tree

Showing 6 changed files with 303 additions and 1 deletion.
diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md
@@ -83,13 +83,22 @@ To run Qdrant on local development environment you need to install below:
 
     ./target/release/qdrant
     ```
+- Install Python dependencies for testing
+    ```shell
+    poetry -C tests install --sync
+    ```
+    Then you could use `poetry -C run pytest tests/openapi` and `poetry -C run pytest tests/consensus_tests` to run the tests.
 - Use the web UI
 
     Web UI repo is [in a separate repo](https://github.com/qdrant/qdrant-web-ui), but there's a utility script to sync it to the `static` folder:
     ```shell
     ./tools/sync-web-ui.sh
     ```
 
+### Nix/NixOS
+If you are using [Nix package manager](https://nixos.org/) (available for Linux and MacOS), you can run `nix-shell` in the project root to get a shell with all dependencies installed.
+It includes dependencies to build Rust code as well as to run Python tests and various tools in the `./tools` directory.
+
 ## Profiling
 
 There are several benchmarks implemented in Qdrant. Benchmarks are not included in CI/CD and might take some time to execute.
@@ -210,7 +219,7 @@ Here is a quick step-by-step guide:
 2. change specs in `/openapi/*ytt.yaml`
 3. add new schema definitions to `src/schema_generator.rs`
 4. run `./tools/generate_openapi_models.sh` to generate specs
-5. update integration tests `tests/openapi` and run them with `poetry -C tests install --sync && poetry -C tests run pytest tests/openapi`
+5. update integration tests `tests/openapi` and run them with `pytest tests/openapi` (use poetry or nix to get `pytest`)
 6. expose file by starting an HTTP server, for instance `python -m http.server`, in `/docs/redoc`
 7. validate specs by browsing redoc on `http://localhost:8000/?v=master`
 8. validate `openapi-merged.yaml` using [swagger editor](https://editor.swagger.io/)

diff --git a/shell.nix b/shell.nix
@@ -0,0 +1,87 @@
+let
+  sources = import ./tools/nix/npins;
+  fenix = import sources.fenix { inherit pkgs; };
+  pkgs = import sources.nixpkgs { };
+  poetry2nix = import sources.poetry2nix { inherit pkgs; };
+
+  versions = builtins.fromJSON (builtins.readFile ./tools/nix/versions.json);
+
+  rust-combined =
+    let
+      stable = fenix.toolchainOf {
+        channel = versions.stable.version;
+        sha256 = versions.stable.sha256;
+      };
+      nightly = fenix.toolchainOf {
+        channel = "nightly";
+        date = versions.nightly.date;
+        sha256 = versions.nightly.sha256;
+      };
+    in
+    fenix.combine [
+      nightly.rustfmt # should be the first
+      stable.rust
+      stable.rust-analyzer
+      stable.rust-src
+    ];
+
+  # A workaround to allow running `cargo +nightly fmt`
+  cargo-wrapper = pkgs.writeScriptBin "cargo" ''
+    #!${pkgs.stdenv.shell}
+    [ "$1" != "+nightly" ] || [ "$2" != "fmt" ] || shift
+    exec ${rust-combined}/bin/cargo "$@"
+  '';
+
+  # Python dependencies used in tests
+  python-env = poetry2nix.mkPoetryEnv {
+    projectDir = ./tests; # reads pyproject.toml and poetry.lock
+    preferWheels = true; # wheels speed up building of the environment
+  };
+
+  # Use mold linker to speed up builds
+  mkShellMold = pkgs.mkShell.override { stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.stdenv; };
+in
+mkShellMold {
+  buildInputs = [
+    # Rust toolchain
+    cargo-wrapper # should be before rust-combined
+    rust-combined
+
+    # Crates' build dependencies
+    pkgs.libunwind # for unwind-sys
+    pkgs.pkg-config # for unwind-sys and other deps
+    pkgs.protobuf # for prost-wkt-types
+    pkgs.rustPlatform.bindgenHook # for bindgen deps
+
+    # For tests and tools
+    pkgs.cargo-nextest # mentioned in .github/workflows/rust.yml
+    pkgs.curl # used in ./tests
+    pkgs.gnuplot # optional runtime dep for criterion
+    pkgs.jq # used in ./tests and ./tools
+    pkgs.nixfmt-rfc-style # to format this file
+    pkgs.npins # used in tools/nix/update.py
+    pkgs.poetry # used to update poetry.lock
+    pkgs.wget # used in tests/storage-compat
+    pkgs.yq-go # used in tools/generate_openapi_models.sh
+    pkgs.ytt # used in tools/generate_openapi_models.sh
+    python-env # used in tests
+  ];
+
+  shellHook = ''
+    # Caching for C/C++ deps, particularly for librocksdb-sys
+    PATH="${pkgs.ccache}/bin:$PATH"
+    export CC="ccache $CC"
+    export CXX="ccache $CXX"
+
+    # Caching for Rust
+    PATH="${pkgs.sccache}/bin:$PATH"
+    export RUSTC_WRAPPER="sccache"
+
+    # Caching for lindera-unidic
+    [ "''${LINDERA_CACHE+x}" ] ||
+      export LINDERA_CACHE="''${XDG_CACHE_HOME:-$HOME/.cache}/lindera"
+
+    # https://qdrant.tech/documentation/guides/common-errors/#too-many-files-open-os-error-24
+    [ "$(ulimit -n)" -ge 10000 ] || ulimit -n 10000
+  '';
+}
diff --git a/tools/nix/npins/default.nix b/tools/nix/npins/default.nix
@@ -0,0 +1,80 @@
+# Generated by npins. Do not modify; will be overwritten regularly
+let
+  data = builtins.fromJSON (builtins.readFile ./sources.json);
+  version = data.version;
+
+  mkSource =
+    spec:
+    assert spec ? type;
+    let
+      path =
+        if spec.type == "Git" then
+          mkGitSource spec
+        else if spec.type == "GitRelease" then
+          mkGitSource spec
+        else if spec.type == "PyPi" then
+          mkPyPiSource spec
+        else if spec.type == "Channel" then
+          mkChannelSource spec
+        else
+          builtins.throw "Unknown source type ${spec.type}";
+    in
+    spec // { outPath = path; };
+
+  mkGitSource =
+    {
+      repository,
+      revision,
+      url ? null,
+      hash,
+      branch ? null,
+      ...
+    }:
+    assert repository ? type;
+    # At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository
+    # In the latter case, there we will always be an url to the tarball
+    if url != null then
+      (builtins.fetchTarball {
+        inherit url;
+        sha256 = hash; # FIXME: check nix version & use SRI hashes
+      })
+    else
+      assert repository.type == "Git";
+      let
+        urlToName =
+          url: rev:
+          let
+            matched = builtins.match "^.*/([^/]*)(\\.git)?$" repository.url;
+
+            short = builtins.substring 0 7 rev;
+
+            appendShort = if (builtins.match "[a-f0-9]*" rev) != null then "-${short}" else "";
+          in
+          "${if matched == null then "source" else builtins.head matched}${appendShort}";
+        name = urlToName repository.url revision;
+      in
+      builtins.fetchGit {
+        url = repository.url;
+        rev = revision;
+        inherit name;
+        # hash = hash;
+      };
+
+  mkPyPiSource =
+    { url, hash, ... }:
+    builtins.fetchurl {
+      inherit url;
+      sha256 = hash;
+    };
+
+  mkChannelSource =
+    { url, hash, ... }:
+    builtins.fetchTarball {
+      inherit url;
+      sha256 = hash;
+    };
+in
+if version == 3 then
+  builtins.mapAttrs (_: mkSource) data.pins
+else
+  throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`"
diff --git a/tools/nix/npins/sources.json b/tools/nix/npins/sources.json
@@ -0,0 +1,35 @@
+{
+  "pins": {
+    "fenix": {
+      "type": "Git",
+      "repository": {
+        "type": "GitHub",
+        "owner": "nix-community",
+        "repo": "fenix"
+      },
+      "branch": "main",
+      "revision": "69c2c0c3c2f56314966dae21d79274515b228482",
+      "url": "https://github.com/nix-community/fenix/archive/69c2c0c3c2f56314966dae21d79274515b228482.tar.gz",
+      "hash": "0pvyjb6dzrnh1rbfzrnbk6507qy0cwip1d2zjmil5rnm3bifdzny"
+    },
+    "nixpkgs": {
+      "type": "Channel",
+      "name": "nixpkgs-unstable",
+      "url": "https://releases.nixos.org/nixpkgs/nixpkgs-24.11pre661631.81610abc161d/nixexprs.tar.xz",
+      "hash": "1cw8psd502wzh7s0nh2nn47y1sk4gl705d23rfgi14xmjj530kg9"
+    },
+    "poetry2nix": {
+      "type": "Git",
+      "repository": {
+        "type": "GitHub",
+        "owner": "nix-community",
+        "repo": "poetry2nix"
+      },
+      "branch": "master",
+      "revision": "8c25e871bba3f472e1569bbf6c0f52dcc34bf2a4",
+      "url": "https://github.com/nix-community/poetry2nix/archive/8c25e871bba3f472e1569bbf6c0f52dcc34bf2a4.tar.gz",
+      "hash": "06cxsh80p6j2n9gnrc0ycc8qap4da75q6l0jwwxaj7dkap7lymg8"
+    }
+  },
+  "version": 3
+}
diff --git a/tools/nix/update.py b/tools/nix/update.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+import argparse
+import hashlib
+import json
+import pathlib
+import re
+import subprocess
+import sys
+import tomllib
+import urllib.request
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Update the versions.json and nix dependencies",
+    )
+    parser.add_argument(
+        "--nix",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="update nix dependencies by running `npins update` (default: yes)",
+    )
+    parser.add_argument(
+        "--nightly",
+        type=str,
+        help="nightly version, e.g. 2021-01-01 (default: use latest)",
+    )
+    parser.add_argument(
+        "--stable",
+        type=str,
+        help="stable version, e.g. 1.76.0 (default: use latest)",
+    )
+    args = parser.parse_args()
+
+    if args.nix:
+        print("Running `npins update`", file=sys.stderr)
+        subprocess.run(
+            args=["npins", "update"],
+            cwd=pathlib.Path(__file__).parent,
+            check=True,
+        )
+
+    print("Updating versions.json", file=sys.stderr)
+    d = f"{args.nightly}/" if args.nightly else ""
+    url = f"https://static.rust-lang.org/dist/{d}channel-rust-nightly.toml"
+    with urllib.request.urlopen(url) as response:
+        data = response.read()
+    nightly_sha256 = hashlib.sha256(data).digest().hex()
+    nightly_date = tomllib.loads(data.decode("utf-8"))["date"]
+
+    d = args.stable if args.stable else "stable"
+    url = f"https://static.rust-lang.org/dist/channel-rust-{d}.toml"
+    with urllib.request.urlopen(url) as response:
+        data = response.read()
+    stable_sha256 = hashlib.sha256(data).digest().hex()
+    stable_version = tomllib.loads(data.decode("utf-8"))["pkg"]["rust"]["version"]
+    m = re.match(r"^(\d+\.\d+\.\d+) \(.*\)$", stable_version)
+    if not m:
+        raise ValueError(f"Failed to parse stable version: {stable_version}")
+    stable_version = m[1]
+
+    result = {
+        "#": "This file is autogenerated by ./update.py",
+        "nightly": {
+            "date": nightly_date,
+            "sha256": nightly_sha256,
+        },
+        "stable": {
+            "version": stable_version,
+            "sha256": stable_sha256,
+        },
+    }
+
+    with open(pathlib.Path(__file__).parent / "versions.json", "w") as f:
+        f.write(json.dumps(result, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/nix/versions.json b/tools/nix/versions.json
@@ -0,0 +1,11 @@
+{
+  "#": "This file is autogenerated by ./update.py",
+  "nightly": {
+    "date": "2024-08-03",
+    "sha256": "e8e1d2f636d1fb068ef9359a9cfa9860adda70e0b8411fa0d33dfdaef9a79984"
+  },
+  "stable": {
+    "version": "1.80.0",
+    "sha256": "e9e37f18ace35528c48463bd16158e6e4445684d497febaa3127509dbf25701e"
+  }
+}