Merge pull request #485 from ingydotnet/ys-workflows

Generate GHA workflow files from originals
biojppm · Jan 9, 2025 · 70c4e36 · 70c4e36
2 parents c183e50 + 728065b
commit 70c4e36
Show file tree

Hide file tree

Showing 46 changed files with 5,019 additions and 1,617 deletions.
diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile
@@ -0,0 +1,62 @@
+SHELL := bash
+
+YS_VERSION := 0.1.87
+YS_PREFIX := /tmp/rapidyaml
+YS := $(YS_PREFIX)/bin/ys-$(YS_VERSION)
+
+INPUT_FILES := $(wildcard */*)
+SOURCE_FILES := $(wildcard *.ys)
+TARGET_FILES := $(SOURCE_FILES:%.ys=%.yml)
+
+export PATH := $(YS_PREFIX)/bin:$(PATH)
+export YSPATH := $(shell pwd -P)/ys
+
+
+build: $(TARGET_FILES)
+
+
+test: force build
+	@git diff --exit-code $(TARGET_FILES) && \
+	  echo -e '\nPASS - No normative changes to .github/workflows/*.yml'
+
+DIFF_ORIG_COMMIT ?= HEAD
+diff:
+	@for s in $(SOURCE_FILES); do \
+	  s=$${s##*/}; \
+	  t=$${s%.yml}; \
+	  [[ $$(git show $(DIFF_ORIG_COMMIT):.github/workflows/.$$t \
+	          2>/dev/null) ]] && t=.$$t; \
+	  diff -u --color=auto \
+	    <(yq -P 'sort_keys(..)' \
+	         -o=props <(git show $(DIFF_ORIG_COMMIT):.github/workflows/$$t) | \
+		 grep -Ev '(^$$|^#)' \
+	     ) \
+	    <(yq -P 'sort_keys(..)' \
+	         -o=props ../workflows/$$s | \
+		 grep -Ev '(^$$|^#)'\
+	     ); \
+	  done
+
+force:
+	touch *.ys
+
+%.yml: %.ys $(YS) $(INPUT_FILES)
+	@echo "$@"
+	if [ -f "$@" ] ; then chmod a+w $@ ; fi
+	@echo "# DO NOT EDIT - GENERATED FROM .github/workflows/$<" > $@
+	@echo >> $@
+	$(YS) -Y $< >> $@
+	@chmod a-w $@
+	@echo "    `wc -ml $<`"
+	@echo "    `wc -ml $@`"
+
+
+# Auto install a specific version of ys
+install-ys: $(YS)
+$(YS):
+	curl -s https://yamlscript.org/install | \
+	  BIN=1 VERSION=$(YS_VERSION) PREFIX=$(YS_PREFIX) bash
+
+stats:
+	@echo "ys : $$(wc -l *.ys)"
+	@echo "yml: $$(wc -l *.yml)"
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -0,0 +1,25 @@
+`.github/workflows/*.ys`
+========================
+
+Use YAMLScript for GHA workflows
+
+
+## Synopsis
+
+```bash
+$ make build
+$ make test
+$ make diff
+```
+
+
+## Description
+
+Edit the `*.ys` files in this directory and run `make build` to generate the
+files used by GitHub Actions (`.*.yml`).
+
+Running `make test` runs `make build` and shows the diff in the of the workflow
+`.foo.yml` files.
+
+If you are only refactoring the `*.ys` source files, you should see no diff
+when you run this.
diff --git a/.github/workflows/bash/benchmarks-install.bash b/.github/workflows/bash/benchmarks-install.bash
@@ -0,0 +1,7 @@
+which python
+which pip
+python --version
+pip --version
+pip install -v -r ext/c4core/cmake/bm-xp/requirements.txt
+python -c 'import munch ; print("ok!") ; exit(0)'
+echo $?
diff --git a/.github/workflows/bash/benchmarks-run.bash b/.github/workflows/bash/benchmarks-run.bash
@@ -0,0 +1,11 @@
+set -x
+source vars.sh
+echo SRC_TAG=$SRC_TAG
+echo SRC_VERSION=$SRC_VERSION
+desc=$SRC_TAG
+for bl in ${{matrix.bitlinks}} ; do
+  dst=$(echo ryml-benchmark_results/$desc/x86_64/${{matrix.cxx}}-${{matrix.bt}}-c++${{matrix.std}}-$bl | sed 's:++-:xx:g' | sed 's:+:x:g')
+  mkdir -p $dst
+  find build -name bm-results
+  mv -vf build/$bl/bm/bm-results/* $dst/.
+done
diff --git a/.github/workflows/bash/clang-checkout.bash b/.github/workflows/bash/clang-checkout.bash
@@ -0,0 +1,11 @@
+set -x
+echo GITHUB_REF=$GITHUB_REF
+echo GITHUB_HEAD_REF=$GITHUB_HEAD_REF
+branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}  # https://stackoverflow.com/questions/58033366
+echo branch=$branch
+git init -q .
+git config --system --add safe.directory '*'  # needed for running in the docker image. see https://github.com/actions/checkout/issues/1169
+git remote add origin $GITHUB_SERVER_URL/$GITHUB_REPOSITORY
+git fetch origin $branch
+git reset --hard FETCH_HEAD
+git submodule update --init --recursive
diff --git a/.github/workflows/bash/install-configure.bash b/.github/workflows/bash/install-configure.bash
@@ -0,0 +1,10 @@
+mkdir -p $BDIR
+mkdir -p $IDIR
+cmake -S ${{matrix.sdir}} -B $BDIR \
+  -DRYML_BUILD_TESTS=ON \
+  -DRYML_VALGRIND=OFF \
+  -DCMAKE_BUILD_TYPE=${{matrix.bt}} \
+  -DCMAKE_INSTALL_PREFIX=$IDIR \
+  ${{matrix.gen}} \
+  ${{matrix.vars}} \
+  ${{matrix.commonvars}}
diff --git a/.github/workflows/bash/install-preinstall.bash b/.github/workflows/bash/install-preinstall.bash
@@ -0,0 +1,6 @@
+if [ "${{matrix.sdir}}" == "test/test_install" ] ; then
+  mkdir -p $BDIR-staging
+  cmake -S . -B $BDIR-staging -DCMAKE_INSTALL_PREFIX=$PDIR -DCMAKE_BUILD_TYPE=${{matrix.bt}} ${{matrix.gen}} ${{matrix.commonvars}}
+  cmake --build $BDIR-staging --config ${{matrix.bt}} --target ${{matrix.tgt}} -j
+  cmake --build $BDIR-staging --config ${{matrix.bt}} --target install
+fi
diff --git a/.github/workflows/bash/rarearchs-test-install.bash b/.github/workflows/bash/rarearchs-test-install.bash
@@ -0,0 +1,23 @@
+set -x
+start_time=$SECONDS
+time apt-get update -y
+time apt-get install -y \
+  git \
+  build-essential
+# arm platforms need an up-to-date cmake:
+# https://gitlab.kitware.com/cmake/cmake/-/issues/20568
+if [ "${{matrix.arch}}" == "armv6" ] || [ "${{matrix.arch}}" == "armv7" ] ; then
+  time apt-get install -y \
+    gpg \
+    wget \
+    apt-transport-https
+  wget --no-check-certificate -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+  echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null
+  time apt-get update -y
+  rm /usr/share/keyrings/kitware-archive-keyring.gpg
+  time apt-get install kitware-archive-keyring
+  time apt-get update -y
+fi
+time apt-get install -y cmake cmake-data
+cmake --version
+echo "install took $((SECONDS - start_time))"
diff --git a/.github/workflows/bash/rarearchs-test-run.bash b/.github/workflows/bash/rarearchs-test-run.bash
@@ -0,0 +1,26 @@
+set -x
+start_time=$SECONDS
+uname -a
+pwd
+ls -lFhp .
+#
+bdir=build_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}}
+idir=install_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}}
+mkdir -p $bdir
+#
+time cmake -S . -B $bdir \
+  -DCMAKE_INSTALL_PREFIX=$idir \
+  -DCMAKE_BUILD_TYPE=${{matrix.bt}} \
+  -DC4_CXX_STANDARD=${{matrix.std}} \
+  -DCXX_STANDARD=${{matrix.std}} \
+  -DRYML_DEV=ON \
+  -DRYML_TEST_SUITE=ON \
+  -DRYML_BUILD_BENCHMARKS=OFF \
+  -DRYML_SANITIZE=OFF \
+  -DRYML_LINT=OFF \
+  -DRYML_VALGRIND=OFF
+#
+time cmake --build $bdir -j 3 --target ryml-test-build
+#
+time cmake --build $bdir -j 3 --target ryml-test-run
+echo "run took $((SECONDS - start_time))"
diff --git a/.github/workflows/bash/release-create-source-packages.bash b/.github/workflows/bash/release-create-source-packages.bash
@@ -0,0 +1,12 @@
+pwd
+ls -lFhp
+source vars.sh
+echo SRC_TAG=$SRC_TAG
+echo SRC_VERSION=$SRC_VERSION
+id=${PROJ_PKG_NAME}${SRC_VERSION}
+name=${id}-src
+mkdir -p assets
+git-archive-all --prefix $name assets/$name.tgz
+git-archive-all --prefix $name assets/$name.zip
+python --version
+python tools/amalgamate.py assets/$id.hpp
diff --git a/.github/workflows/bash/release-normalize.bash b/.github/workflows/bash/release-normalize.bash
@@ -0,0 +1,4 @@
+sdist_orig=`find dist -type f -name 'rapidyaml-*.zip'`
+[ ! -f $sdist_orig ] && exit 1
+sdist=`echo $sdist_orig | sed 's:\.zip:-python_src.zip:'`
+mv -fv $sdist_orig $sdist
diff --git a/.github/workflows/bash/release-package-python.bash b/.github/workflows/bash/release-package-python.bash
@@ -0,0 +1,3 @@
+python --version
+pip install -v -r requirements.txt
+python setup.py sdist --formats=zip
diff --git a/.github/workflows/bash/release-shared-normalize.bash b/.github/workflows/bash/release-shared-normalize.bash
@@ -0,0 +1,7 @@
+set -x
+source vars.sh
+mkdir -p assets
+asset_src=`ls -1 ./build/shared64/*-${{matrix.config.sfxg}}`
+asset_dst=./assets/${PROJ_PKG_NAME}${SRC_VERSION}-${{matrix.config.sfxp}}
+[ ! -f $asset_src ] && exit 1
+cp -fav $asset_src $asset_dst
diff --git a/.github/workflows/bash/release-test-with-python.bash b/.github/workflows/bash/release-test-with-python.bash
@@ -0,0 +1,20 @@
+set -x
+echo "python ${{matrix.config.pythonv}} ${{matrix.config.py_arch}} ${{matrix.config.cibw_arch}}"
+# skip 32 bit tests, as Python 32 bits are not available in ubuntu
+arch="${{matrix.config.cibw_arch}}"
+if [ "$arch" == "x86" ] || [ "$arch" == "i686" ] ; then
+  exit 0
+fi
+python --version
+python -c 'import sys ; import struct ; print("python:", sys.version, struct.calcsize("P") * 8, "bits")'
+pip --version
+pip install -v -r requirements.txt
+pip install -v -r api/python/requirements.txt
+for whl in dist/* ; do
+  pip install -v $whl
+  pip show -f rapidyaml
+  python -c 'import ryml ; print("ryml", ryml.version, ryml.version_tuple)'
+  python -c 'import ryml ; tree = ryml.parse_in_arena(b"{foo: bar}") ; assert tree.key(1) == b"foo" ; assert tree.val(1) == b"bar" ; print(str(tree.key(1), "utf8")) ; print(str(tree.val(1), "utf8"))'
+  python -m pytest -vvv api/python/tests
+  pip uninstall -y -v rapidyaml
+done