openebs-archive · pawanpraka1 · Sep 9, 2020 · Sep 4, 2020 · Sep 5, 2020 · Sep 8, 2020
diff --git a/.travis.yml b/.travis.yml
@@ -9,15 +9,36 @@ env:
   global:
     # Travis limits maximum log size, we have to cut tests output
     - CSTOR_TEST_TRAVIS_LOG_MAX_LENGTH=800
+jobs:
+  include:
+    - os: linux
+      arch: amd64
+      env:
+        - RUN_UZFS_TESTS=1
+        - RELEASE_TAG_DOWNSTREAM=1
+    - os: linux
+      arch: arm64
+      env:
+        - RUN_UZFS_TESTS=0
+        - RELEASE_TAG_DOWNSTREAM=0
+
 before_install:
     - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
     - sudo apt-get update -qq
     - sudo apt-get install --yes -qq gcc-6 g++-6
-    - sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot linux-headers-$(uname -r) libaio-dev jq
+    - sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot libaio-dev jq
+    # linux-header package name is different on arm.
+    - if [ "$TRAVIS_CPU_ARCH" == "arm64" ]; then
+        sudo apt-get install --yes -qq linux-headers-generic;
+      else
+        sudo apt-get install --yes -qq linux-headers-$(uname -r);
+      fi
     - sudo apt-get install --yes -qq zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libssl-dev libjson-c-dev
     - sudo apt-get install --yes -qq lcov libjemalloc-dev
-    # packages for tests
-    - sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio
+    # packages for tests - only on amd64
+    - if [ "$TRAVIS_CPU_ARCH" == "amd64" ]; then
+        sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio;
+      fi
     - sudo apt-get install --yes -qq libgtest-dev cmake
     # packages for debugging
     - sudo apt-get install gdb
@@ -39,7 +60,7 @@ install:
     # we need fio repo to build zfs replica fio engine
     - git clone https://github.com/axboe/fio
     - cd fio
-    - git checkout fio-3.7
+    - git checkout fio-3.9
     - ./configure
     - make -j4
     - cd ..
@@ -50,35 +71,47 @@ install:
     - ./configure
     - make -j4
     - cd ..
-    # we need cstor code
+    # we need cstor code for compiling libcstor(uZFS feature)
     - git clone https://github.com/openebs/cstor.git
     - cd cstor
     - if [ ${TRAVIS_BRANCH} == "master" ]; then git checkout develop; else git checkout ${TRAVIS_BRANCH} || git checkout develop; fi
     - git branch
-    - cd ..
-    # return to libcstor code
-    - cd libcstor
+    # Return to libcstor code base
+    - popd
     - sh autogen.sh
     - ./configure --enable-debug --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include
     - make -j4
     - sudo make install
     - sudo ldconfig
+    # Return to cstor code
     - cd ..
     - cd cstor
     - sh autogen.sh
     - ./configure --with-config=user  --enable-debug --enable-uzfs=yes --with-jemalloc --with-fio=$PWD/../fio --with-libcstor=$PWD/../libcstor/include
     - make -j4;
-    # return to libcstor code to run lint checks
+    # Return to libcstor code to complie zrepl which contains main process and to run lint checks
     - cd ..
     - cd libcstor
+    - make check-license
     - make -f ../cstor/Makefile cstyle CSTORDIR=$PWD/../cstor
+    # Go to zrepl directory to build zrepl related targets
+    - cd cmd/zrepl
+    - make
+    - cd ../../
     # back to cstor for running further tests.
     - cd ..
     - cd cstor
 script:
-    - export FIO_SRCDIR=$PWD/../fio;
-    - sudo bash ./print_debug_info.sh &
-    - sudo bash ../libcstor/tests/cstor/script/test_uzfs.sh -T all || travis_terminate 1;
+    # run ztest and test supported zio backends
+    - if [ $RUN_UZFS_TESTS = 1 ]; then
+        export FIO_SRCDIR=$PWD/../fio;
+        sudo bash ./print_debug_info.sh &
+        sudo bash ../libcstor/tests/cstor/script/test_uzfs.sh -T all || travis_terminate 1;
+      fi
+    # Go back to libcstor to build images
+    - cd ..
+    - cd libcstor
+    - ./build_image.sh || travis_terminate 1;
     # If this build is running due to travis release tag, then
     # go ahead and tag the dependent repo.
     #  
@@ -113,7 +146,7 @@ script:
           REL_SUFFIX="-$TAG_SUFFIX";
         fi;
         REL_BRANCH=$(echo $(echo "$TRAVIS_TAG" | cut -d'-' -f1 | rev | cut -d'.' -f2- | rev).x$REL_SUFFIX) ;
-        ./buildscripts/git-release "$REPO_ORG/cstor" "$TRAVIS_TAG" "$REL_BRANCH" || travis_terminate 1;
+        ./buildscripts/git-release "$REPO_ORG/istgt" "$TRAVIS_TAG" "$REL_BRANCH" || travis_terminate 1;
       fi
 after_failure:
     - find /var/tmp/test_results/current/log -type f -name '*' -printf "%f\n" -exec cut -c -$CSTOR_TEST_TRAVIS_LOG_MAX_LENGTH {} \;

diff --git a/Makefile.am b/Makefile.am
@@ -1,4 +1,28 @@
+# Copyright © 2017-2019 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ACLOCAL_AMFLAGS= -I m4
 AUTOMAKE_OPTIONS = foreign
 
 SUBDIRS = src include
+
+check-license:
+	@echo ">> checking license header"
+	licRes=$$(for file in $$(find . -type f -regex '.*\.sh\|.*\.c\|.*\.cc\|.*\.am\|.*\.h\|.*Docker.*' ! -path ./config.h) ; do \
+	         awk 'NR<=5' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \
+	done); \
+	if [ -n "$${licRes}" ]; then \
+		echo "license header checking failed:"; echo "$${licRes}"; \
+		exit 1; \
+	fi
diff --git a/autogen.sh b/autogen.sh
@@ -1,4 +1,18 @@
 #!/bin/sh
 
+# Copyright © 2017-2019 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 autoreconf -fiv
 rm -Rf autom4te.cache
diff --git a/build_image.sh b/build_image.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+
+# Copyright © 2017-2019 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+pwd
+
+# Determine the arch/os we're building for
+ARCH=$(uname -m)
+
+# Build libcstor
+cd ../libcstor
+make clean
+sh autogen.sh
+./configure --with-zfs-headers=$PWD/../cstor/include --with-spl-headers=$PWD/../cstor/lib/libspl/include
+make -j$(nproc)
+sudo make install
+sudo ldconfig
+
+
+# Build cstor
+cd ../cstor
+make clean
+sh autogen.sh
+./configure --enable-uzfs=yes --with-config=user --with-jemalloc --with-libcstor=$PWD/../libcstor/include
+make clean
+make -j$(nproc)
+
+# Build zrepl target and docker files exist in libcstor
+cd ../libcstor/cmd/zrepl
+make clean
+make
+cd ../../
+
+# The images can be pushed to any docker/image registeries
+# like docker hub, quay. The registries are specified in 
+# the `build/push` script.
+#
+# The images of a project or company can then be grouped
+# or hosted under a unique organization key like `openebs`
+#
+# Each component (container) will be pushed to a unique 
+# repository under an organization. 
+# Putting all this together, an unique uri for a given 
+# image comprises of:
+#   <registry url>/<image org>/<image repo>:<image-tag>
+#
+# IMAGE_ORG can be used to customize the organization 
+# under which images should be pushed. 
+# By default the organization name is `openebs`. 
+
+if [ -z "${IMAGE_ORG}" ]; then
+  IMAGE_ORG="openebs"
+fi
+
+# Specify the date of build
+DBUILD_DATE=$(date +'%Y-%m-%dT%H:%M:%SZ')
+
+# Specify the docker arg for repository url
+if [ -z "${DBUILD_REPO_URL}" ]; then
+  DBUILD_REPO_URL="https://github.com/openebs/cstor"
+fi
+
+# Specify the docker arg for website url
+if [ -z "${DBUILD_SITE_URL}" ]; then
+  DBUILD_SITE_URL="https://openebs.io"
+fi
+
+DBUILD_ARGS="--build-arg DBUILD_DATE=${DBUILD_DATE} --build-arg DBUILD_REPO_URL=${DBUILD_REPO_URL} --build-arg DBUILD_SITE_URL=${DBUILD_SITE_URL} --build-arg ARCH=${ARCH}"
+
+
+if [ "${ARCH}" = "x86_64" ]; then
+	REPO_NAME="$IMAGE_ORG/cstor-base"
+	DOCKERFILE_BASE="Dockerfile.base"
+	DOCKERFILE="Dockerfile"
+elif [ "${ARCH}" = "aarch64" ]; then
+	REPO_NAME="$IMAGE_ORG/cstor-base-arm64"
+	DOCKERFILE_BASE="Dockerfile.base.arm64"
+	DOCKERFILE="Dockerfile.arm64"
+else
+	echo "${ARCH} is not supported"
+	exit 1
+fi
+
+mkdir -p ./docker/zfs/bin
+mkdir -p ./docker/zfs/lib
+
+cp cmd/zrepl/.libs/zrepl ./docker/zfs/bin
+cp ../cstor/cmd/zpool/.libs/zpool ./docker/zfs/bin
+cp ../cstor/cmd/zfs/.libs/zfs ./docker/zfs/bin
+cp ../cstor/cmd/zstreamdump/.libs/zstreamdump ./docker/zfs/bin
+
+cp ../cstor/lib/libzpool/.libs/*.so* ./docker/zfs/lib
+cp ../cstor/lib/libuutil/.libs/*.so* ./docker/zfs/lib
+cp ../cstor/lib/libnvpair/.libs/*.so* ./docker/zfs/lib
+cp ../cstor/lib/libzfs/.libs/*.so* ./docker/zfs/lib
+cp ../cstor/lib/libzfs_core/.libs/*.so* ./docker/zfs/lib
+cp src/.libs/*.so* ./docker/zfs/lib
+
+sudo docker version
+sudo docker build --help
+
+curl --fail https://raw.githubusercontent.com/openebs/charts/gh-pages/scripts/release/buildscripts/push > ./docker/push
+chmod +x ./docker/push
+
+## Building image for cstor-base
+echo "Build image ${REPO_NAME}:ci with BUILD_DATE=${DBUILD_DATE}"
+cd docker && \
+ sudo docker build -f ${DOCKERFILE_BASE} -t ${REPO_NAME}:ci ${DBUILD_ARGS} . && \
+ DIMAGE=${REPO_NAME} ./push && \
+ cd ..
+if [ $? -ne 0 ]; then
+ echo "Failed to run push script for ${REPO_NAME}"
+ exit 1
+fi
+
+if [ "${ARCH}" = "x86_64" ]; then
+	REPO_NAME="$IMAGE_ORG/cstor-pool"
+elif [ "${ARCH}" = "aarch64" ]; then
+	REPO_NAME="$IMAGE_ORG/cstor-pool-arm64"
+fi 
+
+echo "Build image ${REPO_NAME}:ci with BUILD_DATE=${DBUILD_DATE}"
+cd docker && \
+ sudo docker build -f ${DOCKERFILE} -t ${REPO_NAME}:ci ${DBUILD_ARGS} . && \
+ DIMAGE=${REPO_NAME} ./push && \
+ cd ..
+if [ $? -ne 0 ]; then
+ echo "Failed to run push script for ${REPO_NAME}"
+ exit 1
+fi
+
+rm -rf ./docker/zfs
diff --git a/changelogs/unreleased/70-mittachaitu b/changelogs/unreleased/70-mittachaitu
@@ -0,0 +1 @@
+chore(build): build and push images to repositories from libcstor
diff --git a/cmd/Makefile.am b/cmd/Makefile.am
@@ -0,0 +1,15 @@
+# Copyright © 2020 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SUBDIRS = zrepl
diff --git a/cmd/zrepl/.gitignore b/cmd/zrepl/.gitignore
@@ -0,0 +1 @@
+/zrepl
diff --git a/cmd/zrepl/Makefile.am b/cmd/zrepl/Makefile.am
@@ -0,0 +1,48 @@
+# Copyright © 2017-2019 The OpenEBS Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -Wnoformat-truncation to get rid of compiler warning for unchecked
+#  truncating snprintfs on gcc 7.1.1.
+AM_CFLAGS = $(FRAME_LARGER_THAN)
+# Dynamic symbol table is needed for printing symbol names in stack trace
+AM_LDFLAGS = -rdynamic
+
+AM_CPPFLAGS  = -D_GNU_SOURCE -D__EXTENSIONS__ -D_REENTRANT
+AM_CPPFLAGS += -D_POSIX_PTHREAD_SEMANTICS -D_FILE_OFFSET_BITS=64
+AM_CPPFLAGS += -D_LARGEFILE64_SOURCE -DHAVE_LARGE_STACKS=1
+AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\"
+AM_CPPFLAGS += -DLIBEXECDIR=\"$(libexecdir)\"
+AM_CPPFLAGS += -DRUNSTATEDIR=\"$(runstatedir)\"
+AM_CPPFLAGS += -DSBINDIR=\"$(sbindir)\"
+AM_CPPFLAGS += -DSYSCONFDIR=\"$(sysconfdir)\"
+AM_CPPFLAGS += $(CODE_COVERAGE_CPPFLAGS) $(UZFS_CFLAGS)
+
+DEFAULT_INCLUDES = \
+	-I$(top_srcdir)/include	\
+    ${ZFS_SRC}	\
+    ${SPL_SRC}
+
+sbin_PROGRAMS = zrepl
+
+zrepl_SOURCES = \
+	zrepl.c
+
+zrepl_LDADD = \
+     $(zfssrc)/../lib/libnvpair/libnvpair.la \
+     $(zfssrc)/../lib/libuutil/libuutil.la \
+     $(zfssrc)/../lib/libzpool/libzpool.la \
+     $(zfssrc)/../lib/libzfs/libzfs.la \
+     $(zfssrc)/../lib/libzfs_core/libzfs_core.la
+
+zrepl_LDADD += -lcstor -ludev -luuid -lz
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		chore(build): build and push images to repositories from libcstor