openebs-archive · pawanpraka1 · Sep 9, 2020 · Sep 4, 2020 · Sep 5, 2020 · Sep 8, 2020
diff --git a/.travis.yml b/.travis.yml
@@ -9,15 +9,36 @@ env:
   global:
     # Travis limits maximum log size, we have to cut tests output
     - CSTOR_TEST_TRAVIS_LOG_MAX_LENGTH=800
+jobs:
+  include:
+    - os: linux
+      arch: amd64
+      env:
+        - RUN_UZFS_TESTS=1
+        - RELEASE_TAG_DOWNSTREAM=1
+    - os: linux
+      arch: arm64
+      env:
+        - RUN_UZFS_TESTS=0
+        - RELEASE_TAG_DOWNSTREAM=0
+
 before_install:
     - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
     - sudo apt-get update -qq
     - sudo apt-get install --yes -qq gcc-6 g++-6
-    - sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot linux-headers-$(uname -r) libaio-dev jq
+    - sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot libaio-dev jq
+    # linux-header package name is different on arm.
+    - if [ "$TRAVIS_CPU_ARCH" == "arm64" ]; then
+        sudo apt-get install --yes -qq linux-headers-generic;
+      else
+        sudo apt-get install --yes -qq linux-headers-$(uname -r);
+      fi
     - sudo apt-get install --yes -qq zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libssl-dev libjson-c-dev
     - sudo apt-get install --yes -qq lcov libjemalloc-dev
-    # packages for tests
-    - sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio
+    # packages for tests - only on amd64
+    - if [ "$TRAVIS_CPU_ARCH" == "amd64" ]; then
+        sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server;
+      fi
     - sudo apt-get install --yes -qq libgtest-dev cmake
     # packages for debugging
     - sudo apt-get install gdb
@@ -39,7 +60,7 @@ install:
     # we need fio repo to build zfs replica fio engine
     - git clone https://github.com/axboe/fio
     - cd fio
-    - git checkout fio-3.7
+    - git checkout fio-3.9
     - ./configure
     - make -j4
     - cd ..
@@ -50,35 +71,46 @@ install:
     - ./configure
     - make -j4
     - cd ..
-    # we need cstor code
+    # we need cstor code for compiling libcstor(uZFS feature)
     - git clone https://github.com/openebs/cstor.git
     - cd cstor
     - if [ ${TRAVIS_BRANCH} == "master" ]; then git checkout develop; else git checkout ${TRAVIS_BRANCH} || git checkout develop; fi
     - git branch
-    - cd ..
-    # return to libcstor code
-    - cd libcstor
+    # Return to libcstor code base
+    - popd
     - sh autogen.sh
     - ./configure --enable-debug --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include
     - make -j4
     - sudo make install
     - sudo ldconfig
+    # Return to cstor code
     - cd ..
     - cd cstor
     - sh autogen.sh
     - ./configure --with-config=user  --enable-debug --enable-uzfs=yes --with-jemalloc --with-fio=$PWD/../fio --with-libcstor=$PWD/../libcstor/include
     - make -j4;
-    # return to libcstor code to run lint checks
+    # Return to libcstor code to complie zrepl which contains main process and to run lint checks
     - cd ..
     - cd libcstor
+    - make check-license
     - make -f ../cstor/Makefile cstyle CSTORDIR=$PWD/../cstor
+    # Go to zrepl directory to build zrepl related targets
+    - cd cmd/zrepl
+    - make
+    - cd ../../
     # back to cstor for running further tests.
     - cd ..
     - cd cstor
 script:
-    - export FIO_SRCDIR=$PWD/../fio;
-    - sudo bash ./print_debug_info.sh &
-    - sudo bash ../libcstor/tests/cstor/script/test_uzfs.sh -T all || travis_terminate 1;
+    # run ztest and test supported zio backends
+    - if [ $RUN_UZFS_TESTS = 1 ]; then
+        export FIO_SRCDIR=$PWD/../fio;
+        sudo bash ./print_debug_info.sh &
+        sudo bash ../libcstor/tests/cstor/script/test_uzfs.sh -T all || travis_terminate 1;
+      fi
+    # Go back to libcstor to build images
+    - cd ../libcstor
+    - ./build_image.sh || travis_terminate 1;
     # If this build is running due to travis release tag, then
     # go ahead and tag the dependent repo.
     #  
@@ -113,7 +145,7 @@ script:
           REL_SUFFIX="-$TAG_SUFFIX";
         fi;
         REL_BRANCH=$(echo $(echo "$TRAVIS_TAG" | cut -d'-' -f1 | rev | cut -d'.' -f2- | rev).x$REL_SUFFIX) ;
-        ./buildscripts/git-release "$REPO_ORG/cstor" "$TRAVIS_TAG" "$REL_BRANCH" || travis_terminate 1;
+        ./buildscripts/git-release "$REPO_ORG/istgt" "$TRAVIS_TAG" "$REL_BRANCH" || travis_terminate 1;
       fi
 after_failure:
     - find /var/tmp/test_results/current/log -type f -name '*' -printf "%f\n" -exec cut -c -$CSTOR_TEST_TRAVIS_LOG_MAX_LENGTH {} \;

diff --git a/Makefile.am b/Makefile.am
@@ -1,4 +1,28 @@
+# Copyright © 2020 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ACLOCAL_AMFLAGS= -I m4
 AUTOMAKE_OPTIONS = foreign
 
 SUBDIRS = src include
+
+check-license:
+	@echo ">> checking license header"
+	licRes=$$(for file in $$(find . -type f -regex '.*\.sh\|.*\.c\|.*\.cc\|.*\.am\|.*\.h\|.*Docker.*' ! -path ./config.h) ; do \
+	         awk 'NR<=5' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \
+	done); \
+	if [ -n "$${licRes}" ]; then \
+		echo "license header checking failed:"; echo "$${licRes}"; \
+		exit 1; \
+	fi
diff --git a/RELEASE.md b/RELEASE.md
@@ -18,15 +18,28 @@ Once all the above tests are completed, a main release tagged image is published
 
 ## Release Tagging
 
-libcstor is released as a library part of [github/cstor](https://github.com/openebs/cstor) container image with a versioned tag.
+libcstor is released as a container image with a versioned tag.
 
 Before creating a release, the repo owner needs to create a separate branch from the active branch, which is `master`. Name of the branch should follow the naming convention of `v.1.9.x` if release is for v1.9.0.
 
 Once the release branch is created, changelog from `changelogs/unreleased` needs to be moved to release specific folder `changelogs/v1.9.x`, if release branch is `v1.10.x` then folder will be `changelogs/v1.10.x`.
 
 The format of the release tag is either "Release-Name-RC1" or "Release-Name" depending on whether the tag is a release candidate or a release. (Example: v1.9.0-RC1 is a GitHub release tag for libcstor release build. v1.9.0 is the release tag that is created after the release criteria are satisfied by the libcstor builds.)
 
-Once the release is triggered, Travis build process has to be monitored. Since libcstor is a library and it is published as part of [openebs/cstor](https://github.com/openebs/cstor) images, release process from libcstor repo won't publish any images.
+Once the release is triggered, Travis build process has to be monitored. Once Travis build is passed images are pushed to docker hub and quay.io. Images can be verified by going through docker hub and quay.io. Also the images shouldn't have any high level vulnerabilities.
+
+Images are published at the following location:
+For AMD64:
+```
+https://quay.io/repository/openebs/cstor-pool?tab=tags
+https://hub.docker.com/r/openebs/cstor-pool/tags
+```
+
+For ARM64:
+```
+https://quay.io/repository/openebs/cstor-pool-arm64?tab=tags
+https://hub.docker.com/r/openebs/cstor-pool-arm64/tags
+```
 
 Once a release is created, update the release description with the changelog mentioned in folder `changelog/v1.9.x`. Once the changelogs are updated in the release, the repo owner needs to create a PR to `master` with the following details:
 1. update the changelog from folder `changelog/v1.9.x` to file `libcstor/CHANGELOG-v1.9.md`

diff --git a/autogen.sh b/autogen.sh
@@ -1,4 +1,18 @@
 #!/bin/sh
 
+# Copyright © 2019 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 autoreconf -fiv
 rm -Rf autom4te.cache
diff --git a/build_image.sh b/build_image.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# Copyright © 2020 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+pwd
+
+# Determine the arch/os we're building for
+ARCH=$(uname -m)
+
+# Build libcstor
+make clean
+sh autogen.sh
+./configure --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include
+make -j$(nproc)
+sudo make install
+sudo ldconfig
+
+# Build cstor
+cd ../cstor
+make clean
+sh autogen.sh
+./configure --enable-uzfs=yes --with-config=user --with-jemalloc --with-libcstor=$PWD/../libcstor/include
+make clean
+make -j$(nproc)
+
+# Build zrepl binary
+cd ../libcstor/cmd/zrepl
+make clean
+make
+cd ../../
+
+# The images can be pushed to any docker/image registeries
+# like docker hub, quay. The registries are specified in 
+# the `build/push` script.
+#
+# The images of a project or company can then be grouped
+# or hosted under a unique organization key like `openebs`
+#
+# Each component (container) will be pushed to a unique 
+# repository under an organization. 
+# Putting all this together, an unique uri for a given 
+# image comprises of:
+#   <registry url>/<image org>/<image repo>:<image-tag>
+#
+# IMAGE_ORG can be used to customize the organization 
+# under which images should be pushed. 
+# By default the organization name is `openebs`. 
+
+if [ -z "${IMAGE_ORG}" ]; then
+  IMAGE_ORG="openebs"
+fi
+
+# Specify the date of build
+DBUILD_DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
+
+# Specify the docker arg for repository url
+if [ -z "${DBUILD_REPO_URL}" ]; then
+  DBUILD_REPO_URL="https://github.com/openebs/libcstor"
+fi
+
+# Specify the docker arg for website url
+if [ -z "${DBUILD_SITE_URL}" ]; then
+  DBUILD_SITE_URL="https://openebs.io"
+fi
+
+DBUILD_ARGS="--build-arg DBUILD_DATE=${DBUILD_DATE} --build-arg DBUILD_REPO_URL=${DBUILD_REPO_URL} --build-arg DBUILD_SITE_URL=${DBUILD_SITE_URL} --build-arg ARCH=${ARCH}"
+
+
+if [ "${ARCH}" = "x86_64" ]; then
+	REPO_NAME="$IMAGE_ORG/cstor-base"
+	DOCKERFILE_BASE="Dockerfile.base"
+	DOCKERFILE="Dockerfile"
+elif [ "${ARCH}" = "aarch64" ]; then
+	REPO_NAME="$IMAGE_ORG/cstor-base-arm64"
+	DOCKERFILE_BASE="Dockerfile.base.arm64"
+	DOCKERFILE="Dockerfile.arm64"
+else
+	echo "${ARCH} is not supported"
+	exit 1
+fi
+
+mkdir -p ./docker/zfs/bin
+mkdir -p ./docker/zfs/lib
+
+cp cmd/zrepl/.libs/zrepl ./docker/zfs/bin
+cp ../cstor/cmd/zpool/.libs/zpool ./docker/zfs/bin
+cp ../cstor/cmd/zfs/.libs/zfs ./docker/zfs/bin
+cp ../cstor/cmd/zstreamdump/.libs/zstreamdump ./docker/zfs/bin
+
+cp ../cstor/lib/libzpool/.libs/*.so* ./docker/zfs/lib
+cp ../cstor/lib/libuutil/.libs/*.so* ./docker/zfs/lib
+cp ../cstor/lib/libnvpair/.libs/*.so* ./docker/zfs/lib
+cp ../cstor/lib/libzfs/.libs/*.so* ./docker/zfs/lib
+cp ../cstor/lib/libzfs_core/.libs/*.so* ./docker/zfs/lib
+cp src/.libs/*.so* ./docker/zfs/lib
+
+sudo docker version
+sudo docker build --help
+
+curl --fail https://raw.githubusercontent.com/openebs/charts/gh-pages/scripts/release/buildscripts/push > ./docker/push
+chmod +x ./docker/push
+
+## Building image for cstor-base
+echo "Build image ${REPO_NAME}:ci with BUILD_DATE=${DBUILD_DATE}"
+cd docker && \
+ sudo docker build -f ${DOCKERFILE_BASE} -t ${REPO_NAME}:ci ${DBUILD_ARGS} . && \
+ DIMAGE=${REPO_NAME} ./push && \
+ cd ..
+
+if [ "${ARCH}" = "x86_64" ]; then
+	REPO_NAME="$IMAGE_ORG/cstor-pool"
+elif [ "${ARCH}" = "aarch64" ]; then
+	REPO_NAME="$IMAGE_ORG/cstor-pool-arm64"
+fi 
+
+echo "Build image ${REPO_NAME}:ci with BUILD_DATE=${DBUILD_DATE}"
+cd docker && \
+ sudo docker build -f ${DOCKERFILE} -t ${REPO_NAME}:ci ${DBUILD_ARGS} . && \
+ DIMAGE=${REPO_NAME} ./push && \
+ cd ..
+
+rm -rf ./docker/zfs
diff --git a/changelogs/unreleased/70-mittachaitu b/changelogs/unreleased/70-mittachaitu
@@ -0,0 +1 @@
+chore(build): build and push images to repositories from libcstor
diff --git a/cmd/Makefile.am b/cmd/Makefile.am
@@ -0,0 +1,15 @@
+# Copyright © 2020 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SUBDIRS = zrepl
diff --git a/cmd/zrepl/.gitignore b/cmd/zrepl/.gitignore
@@ -0,0 +1 @@
+/zrepl
diff --git a/cmd/zrepl/Makefile.am b/cmd/zrepl/Makefile.am
@@ -0,0 +1,48 @@
+# Copyright © 2020 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -Wnoformat-truncation to get rid of compiler warning for unchecked
+#  truncating snprintfs on gcc 7.1.1.
+AM_CFLAGS = $(FRAME_LARGER_THAN)
+# Dynamic symbol table is needed for printing symbol names in stack trace
+AM_LDFLAGS = -rdynamic
+
+AM_CPPFLAGS  = -D_GNU_SOURCE -D__EXTENSIONS__ -D_REENTRANT
+AM_CPPFLAGS += -D_POSIX_PTHREAD_SEMANTICS -D_FILE_OFFSET_BITS=64
+AM_CPPFLAGS += -D_LARGEFILE64_SOURCE -DHAVE_LARGE_STACKS=1
+AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\"
+AM_CPPFLAGS += -DLIBEXECDIR=\"$(libexecdir)\"
+AM_CPPFLAGS += -DRUNSTATEDIR=\"$(runstatedir)\"
+AM_CPPFLAGS += -DSBINDIR=\"$(sbindir)\"
+AM_CPPFLAGS += -DSYSCONFDIR=\"$(sysconfdir)\"
+AM_CPPFLAGS += $(CODE_COVERAGE_CPPFLAGS) $(UZFS_CFLAGS)
+
+DEFAULT_INCLUDES = \
+	-I$(top_srcdir)/include	\
+    ${ZFS_SRC}	\
+    ${SPL_SRC}
+
+sbin_PROGRAMS = zrepl
+
+zrepl_SOURCES = \
+	zrepl.c
+
+zrepl_LDADD = \
+     $(zfssrc)/../lib/libnvpair/libnvpair.la \
+     $(zfssrc)/../lib/libuutil/libuutil.la \
+     $(zfssrc)/../lib/libzpool/libzpool.la \
+     $(zfssrc)/../lib/libzfs/libzfs.la \
+     $(zfssrc)/../lib/libzfs_core/libzfs_core.la
+
+zrepl_LDADD += -lcstor -ludev -luuid -lz
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		chore(build): build and push images to repositories from libcstor