diff --git a/README.md b/README.md index 1ab36d1..2996d90 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ This will create a few files: * `packages.nix`: Nix expressions for the packages and artifacts. * `common.nix`: common code for building your environment. Please don't edit this by hand. * `default.nix`: the main entry point. You can edit some settings in here. -* `fetchgit`: this contains a patched version of the Nix fetchgit function, necessary until [this PR](https://github.com/NixOS/nixpkgs/pull/104714) lands. > You should make sure the `baseJulia` in `default.nix` matches the Julia version you used in Step 1. diff --git a/julia2nix b/julia2nix index bd5f453..2c60642 100755 --- a/julia2nix +++ b/julia2nix @@ -25,12 +25,6 @@ def main(): # Always copy the common code shutil.copy(script_dir.joinpath("templates").joinpath("common.nix"), output_dir) - # Always copy the modified fetchgit - fetchgit_dest = output_dir.joinpath("fetchgit") - if fetchgit_dest.exists(): - shutil.rmtree(fetchgit_dest) - shutil.copytree(script_dir.joinpath("templates").joinpath("fetchgit"), fetchgit_dest) - # Only copy default.nix if it doesn't exist yet, since the user is meant to modify it if not output_dir.joinpath("default.nix").exists(): shutil.copy(script_dir.joinpath("templates").joinpath("default.nix"), output_dir) diff --git a/templates/common.nix b/templates/common.nix index 305aa46..98575f6 100644 --- a/templates/common.nix +++ b/templates/common.nix @@ -5,6 +5,7 @@ git, stdenvNoCC, cacert, + fetchgit, jq, julia, lib, @@ -21,10 +22,6 @@ }: let - # We need to use a specially modified fetchgit that understands tree hashes, until - # https://github.com/NixOS/nixpkgs/pull/104714 lands - fetchgit = callPackage ./fetchgit {}; - packages = callPackage ./packages.nix {}; ### Repoify packages diff --git a/templates/fetchgit/builder.sh b/templates/fetchgit/builder.sh deleted file mode 100644 index 6ae4646..0000000 --- a/templates/fetchgit/builder.sh +++ /dev/null @@ -1,16 +0,0 @@ -# tested so far with: -# - no revision specified and remote has a HEAD which is used -# - revision specified and remote has a HEAD -# - revision specified and remote without HEAD -source $stdenv/setup - -header "exporting $url (rev $rev) into $out" - -$SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" \ - ${leaveDotGit:+--leave-dotGit} \ - ${deepClone:+--deepClone} \ - ${fetchSubmodules:+--fetch-submodules} \ - ${branchName:+--branch-name "$branchName"} - -runHook postFetch -stopNest diff --git a/templates/fetchgit/default.nix b/templates/fetchgit/default.nix deleted file mode 100644 index 0405951..0000000 --- a/templates/fetchgit/default.nix +++ /dev/null @@ -1,71 +0,0 @@ -{stdenvNoCC, git, cacert}: let - urlToName = url: rev: let - inherit (stdenvNoCC.lib) removeSuffix splitString last; - base = last (splitString ":" (baseNameOf (removeSuffix "/" url))); - - matched = builtins.match "(.*).git" base; - - short = builtins.substring 0 7 rev; - - appendShort = if (builtins.match "[a-f0-9]*" rev) != null - then "-${short}" - else ""; - in "${if matched == null then base else builtins.head matched}${appendShort}"; -in -{ url, rev ? "HEAD", md5 ? "", sha256 ? "", leaveDotGit ? deepClone -, fetchSubmodules ? true, deepClone ? false -, branchName ? null -, name ? urlToName url rev -, # Shell code executed after the file has been fetched - # successfully. This can do things like check or transform the file. - postFetch ? "" -, preferLocalBuild ? true -}: - -/* NOTE: - fetchgit has one problem: git fetch only works for refs. - This is because fetching arbitrary (maybe dangling) commits may be a security risk - and checking whether a commit belongs to a ref is expensive. This may - change in the future when some caching is added to git (?) - Usually refs are either tags (refs/tags/*) or branches (refs/heads/*) - Cloning branches will make the hash check fail when there is an update. - But not all patches we want can be accessed by tags. - - The workaround is getting the last n commits so that it's likely that they - still contain the hash we want. - - for now : increase depth iteratively (TODO) - - real fix: ask git folks to add a - git fetch $HASH contained in $BRANCH - facility because checking that $HASH is contained in $BRANCH is less - expensive than fetching --depth $N. - Even if git folks implemented this feature soon it may take years until - server admins start using the new version? -*/ - -assert deepClone -> leaveDotGit; - -if md5 != "" then - throw "fetchgit does not support md5 anymore, please use sha256" -else -stdenvNoCC.mkDerivation { - inherit name; - builder = ./builder.sh; - fetcher = ./nix-prefetch-git; # This must be a string to ensure it's called with bash. - nativeBuildInputs = [git]; - - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - outputHash = sha256; - - inherit url rev leaveDotGit fetchSubmodules deepClone branchName postFetch; - - GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - - impureEnvVars = stdenvNoCC.lib.fetchers.proxyImpureEnvVars ++ [ - "GIT_PROXY_COMMAND" "SOCKS_SERVER" - ]; - - inherit preferLocalBuild; -} diff --git a/templates/fetchgit/nix-prefetch-git b/templates/fetchgit/nix-prefetch-git deleted file mode 100755 index 7ece68d..0000000 --- a/templates/fetchgit/nix-prefetch-git +++ /dev/null @@ -1,459 +0,0 @@ -#! /usr/bin/env bash - -set -e -o pipefail - -url= -rev= -expHash= -hashType=$NIX_HASH_ALGO -deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE -leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT -fetchSubmodules= -builder= -branchName=$NIX_PREFETCH_GIT_BRANCH_NAME - -# ENV params -out=${out:-} -http_proxy=${http_proxy:-} - -# populated by clone_user_rev() -fullRev= -humanReadableRev= -commitDate= -commitDateStrict8601= - -if test -n "$deepClone"; then - deepClone=true -else - deepClone= -fi - -if test "$leaveDotGit" != 1; then - leaveDotGit= -else - leaveDotGit=true -fi - -usage(){ - echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]] - -Options: - --out path Path where the output would be stored. - --url url Any url understood by 'git clone'. - --rev ref Any sha1 or references (such as refs/heads/master) - --hash h Expected hash. - --branch-name Branch name to check out into - --deepClone Clone the entire repository. - --no-deepClone Make a shallow clone of just the required ref. - --leave-dotGit Keep the .git directories. - --fetch-submodules Fetch submodules. - --builder Clone as fetchgit does, but url, rev, and out option are mandatory. - --quiet Only print the final json summary. -" - exit 1 -} - -# some git commands print to stdout, which would contaminate our JSON output -clean_git(){ - git "$@" >&2 -} - -argi=0 -argfun="" -for arg; do - if test -z "$argfun"; then - case $arg in - --out) argfun=set_out;; - --url) argfun=set_url;; - --rev) argfun=set_rev;; - --hash) argfun=set_hashType;; - --branch-name) argfun=set_branchName;; - --deepClone) deepClone=true;; - --quiet) QUIET=true;; - --no-deepClone) deepClone=;; - --leave-dotGit) leaveDotGit=true;; - --fetch-submodules) fetchSubmodules=true;; - --builder) builder=true;; - -h|--help) usage; exit;; - *) - : $((++argi)) - case $argi in - 1) url=$arg;; - 2) rev=$arg;; - 3) expHash=$arg;; - *) exit 1;; - esac - ;; - esac - else - case $argfun in - set_*) - var=${argfun#set_} - eval $var=$arg - ;; - esac - argfun="" - fi -done - -if test -z "$url"; then - usage -fi - - -init_remote(){ - local url=$1 - clean_git init - clean_git remote add origin "$url" - ( [ -n "$http_proxy" ] && clean_git config http.proxy "$http_proxy" ) || true -} - -# Return the reference of an hash if it exists on the remote repository. -ref_from_hash(){ - local hash=$1 - git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}" -} - -# Return the hash of a reference if it exists on the remote repository. -hash_from_ref(){ - local ref=$1 - git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}" -} - -# Returns a name based on the url and reference -# -# This function needs to be in sync with nix's fetchgit implementation -# of urlToName() to re-use the same nix store paths. -url_to_name(){ - local url=$1 - local ref=$2 - local base - base=$(basename "$url" .git | cut -d: -f2) - - if [[ $ref =~ ^[a-z0-9]+$ ]]; then - echo "$base-${ref:0:7}" - else - echo "$base" - fi -} - -# Fetch everything and checkout the right sha1 -checkout_hash(){ - local hash="$1" - local ref="$2" - - if test -z "$hash"; then - hash=$(hash_from_ref "$ref") - fi - - clean_git fetch -t ${builder:+--progress} origin || return 1 - - local object_type=$(git cat-file -t "$hash") - if [[ "$object_type" == "commit" ]]; then - clean_git checkout -b "$branchName" "$hash" || return 1 - elif [[ "$object_type" == "tree" ]]; then - clean_git config user.email "nix-prefetch-git@localhost" - clean_git config user.name "nix-prefetch-git" - commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash") - clean_git checkout -b "$branchName" "$commit_id" || return 1 - else - echo "Unrecognized git object type: $object_type" - return 1 - fi -} - -# Fetch only a branch/tag and checkout it. -checkout_ref(){ - local hash="$1" - local ref="$2" - - if [[ -n "$deepClone" ]]; then - # The caller explicitly asked for a deep clone. Deep clones - # allow "git describe" and similar tools to work. See - # https://marc.info/?l=nix-dev&m=139641582514772 - # for a discussion. - return 1 - fi - - if test -z "$ref"; then - ref=$(ref_from_hash "$hash") - fi - - if test -n "$ref"; then - # --depth option is ignored on http repository. - clean_git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1 - clean_git checkout -b "$branchName" FETCH_HEAD || return 1 - else - return 1 - fi -} - -# Update submodules -init_submodules(){ - # Add urls into .git/config file - clean_git submodule init - - # list submodule directories and their hashes - git submodule status | - while read -r l; do - local hash - local dir - local name - local url - - # checkout each submodule - hash=$(echo "$l" | awk '{print $1}' | tr -d '-') - dir=$(echo "$l" | sed -n 's/^.[0-9a-f]\+ \(.*[^)]*\)\( (.*)\)\?$/\1/p') - name=$( - git config -f .gitmodules --get-regexp submodule\..*\.path | - sed -n "s,^\(.*\)\.path $dir\$,\\1,p") - url=$(git config --get "${name}.url") - - clone "$dir" "$url" "$hash" "" - done -} - -clone(){ - local top=$PWD - local dir="$1" - local url="$2" - local hash="$3" - local ref="$4" - - cd "$dir" - - # Initialize the repository. - init_remote "$url" - - # Download data from the repository. - checkout_ref "$hash" "$ref" || - checkout_hash "$hash" "$ref" || ( - echo 1>&2 "Unable to checkout $hash$ref from $url." - exit 1 - ) - - # Checkout linked sources. - if test -n "$fetchSubmodules"; then - init_submodules - fi - - if [ -z "$builder" ] && [ -f .topdeps ]; then - if tg help &>/dev/null; then - echo "populating TopGit branches..." - tg remote --populate origin - else - echo "WARNING: would populate TopGit branches but TopGit is not available" >&2 - echo "WARNING: install TopGit to fix the problem" >&2 - fi - fi - - cd "$top" -} - -# Remove all remote branches, remove tags not reachable from HEAD, do a full -# repack and then garbage collect unreferenced objects. -make_deterministic_repo(){ - local repo="$1" - - # run in sub-shell to not touch current working directory - ( - cd "$repo" - # Remove files that contain timestamps or otherwise have non-deterministic - # properties. - rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \ - .git/refs/remotes/origin/HEAD .git/config - - # Remove all remote branches. - git branch -r | while read -r branch; do - clean_git branch -rD "$branch" - done - - # Remove tags not reachable from HEAD. If we're exactly on a tag, don't - # delete it. - maybe_tag=$(git tag --points-at HEAD) - git tag --contains HEAD | while read -r tag; do - if [ "$tag" != "$maybe_tag" ]; then - clean_git tag -d "$tag" - fi - done - - # Do a full repack. Must run single-threaded, or else we lose determinism. - clean_git config pack.threads 1 - clean_git repack -A -d -f - rm -f .git/config - - # Garbage collect unreferenced objects. - # Note: --keep-largest-pack prevents non-deterministic ordering of packs - # listed in .git/objects/info/packs by only using a single pack - clean_git gc --prune=all --keep-largest-pack - ) -} - - -clone_user_rev() { - local dir="$1" - local url="$2" - local rev="${3:-HEAD}" - - # Perform the checkout. - case "$rev" in - HEAD|refs/*) - clone "$dir" "$url" "" "$rev" 1>&2;; - *) - if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then - clone "$dir" "$url" "$rev" "" 1>&2 - else - # if revision is not hexadecimal it might be a tag - clone "$dir" "$url" "" "refs/tags/$rev" 1>&2 - fi;; - esac - - pushd "$dir" >/dev/null - fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1) - humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --) - commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev") - commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev") - popd >/dev/null - - # Allow doing additional processing before .git removal - eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK" - if test -z "$leaveDotGit"; then - echo "removing \`.git'..." >&2 - find "$dir" -name .git -print0 | xargs -0 rm -rf - else - find "$dir" -name .git | while read -r gitdir; do - make_deterministic_repo "$(readlink -f "$gitdir/..")" - done - fi -} - -exit_handlers=() - -run_exit_handlers() { - exit_status=$? - for handler in "${exit_handlers[@]}"; do - eval "$handler $exit_status" - done -} - -trap run_exit_handlers EXIT - -quiet_exit_handler() { - exec 2>&3 3>&- - if [ $1 -ne 0 ]; then - cat "$errfile" >&2 - fi - rm -f "$errfile" -} - -quiet_mode() { - errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")" - exit_handlers+=(quiet_exit_handler) - exec 3>&2 2>"$errfile" -} - -json_escape() { - local s="$1" - s="${s//\\/\\\\}" # \ - s="${s//\"/\\\"}" # " - s="${s//^H/\\\b}" # \b (backspace) - s="${s//^L/\\\f}" # \f (form feed) - s="${s// -/\\\n}" # \n (newline) - s="${s//^M/\\\r}" # \r (carriage return) - s="${s// /\\t}" # \t (tab) - echo "$s" -} - -print_results() { - hash="$1" - if ! test -n "$QUIET"; then - echo "" >&2 - echo "git revision is $fullRev" >&2 - if test -n "$finalPath"; then - echo "path is $finalPath" >&2 - fi - echo "git human-readable version is $humanReadableRev" >&2 - echo "Commit date is $commitDate" >&2 - if test -n "$hash"; then - echo "hash is $hash" >&2 - fi - fi - if test -n "$hash"; then - cat < /dev/null; then - finalPath= - fi - hash=$expHash - fi - - # If we don't know the hash or a path with that hash doesn't exist, - # download the file and add it to the store. - if test -z "$finalPath"; then - - tmpPath="$(mktemp -d "${TMPDIR:-/tmp}/git-checkout-tmp-XXXXXXXX")" - exit_handlers+=(remove_tmpPath) - - tmpFile="$tmpPath/$(url_to_name "$url" "$rev")" - mkdir -p "$tmpFile" - - # Perform the checkout. - clone_user_rev "$tmpFile" "$url" "$rev" - - # Compute the hash. - hash=$(nix-hash --type $hashType --base32 "$tmpFile") - - # Add the downloaded file to the Nix store. - finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile") - - if test -n "$expHash" -a "$expHash" != "$hash"; then - echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2 - exit 1 - fi - fi - - print_results "$hash" - - if test -n "$PRINT_PATH"; then - echo "$finalPath" - fi -fi