Merge pull request grpc#14691 from jtattermusch/kokoro_packages_distr…

…ibtest Add kokoro build_packages and distribtests jobs.
murgatroid99 · Mar 14, 2018 · 261448e · 261448e
2 parents 43911f2 + 80d54f3
commit 261448e
Show file tree

Hide file tree

Showing 18 changed files with 346 additions and 8 deletions.
diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat
@@ -19,17 +19,21 @@ set VERSION=1.11.0-dev
 set NUGET=C:\nuget\nuget.exe
 set DOTNET=dotnet
 
-set -ex
-
 mkdir ..\..\artifacts
 
 @rem Collect the artifacts built by the previous build step if running on Jenkins
 mkdir nativelibs
+@rem Jenkins flow (deprecated)
 powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs"
+@rem Kokoro flow
+powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs"
 
 @rem Collect protoc artifacts built by the previous build step
 mkdir protoc_plugins
+@rem Jenkins flow (deprecated)
 powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins"
+@rem Kokoro flow
+powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins"
 
 %DOTNET% restore Grpc.sln || goto :error
 

diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh
@@ -21,11 +21,17 @@ mkdir -p ../../artifacts/
 
 # Collect the artifacts built by the previous build step
 mkdir -p nativelibs
+# Jenkins flow (deprecated)
 cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true
+# Kokoro flow
+cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true
 
 # Collect protoc artifacts built by the previous build step
 mkdir -p protoc_plugins
+# Jenkins flow (deprecated)
 cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true
+# Kokoro flow
+cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true
 
 dotnet restore Grpc.sln
 

diff --git a/templates/src/csharp/build_packages_dotnetcli.bat.template b/templates/src/csharp/build_packages_dotnetcli.bat.template
@@ -21,17 +21,21 @@
   set NUGET=C:\nuget\nuget.exe
   set DOTNET=dotnet
 
-  set -ex
-
   mkdir ..\..\artifacts
 
   @rem Collect the artifacts built by the previous build step if running on Jenkins
   mkdir nativelibs
+  @rem Jenkins flow (deprecated)
   powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs"
+  @rem Kokoro flow
+  powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs"
 
   @rem Collect protoc artifacts built by the previous build step
   mkdir protoc_plugins
+  @rem Jenkins flow (deprecated)
   powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins"
+  @rem Kokoro flow
+  powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins"
 
   %%DOTNET% restore Grpc.sln || goto :error
 

diff --git a/templates/src/csharp/build_packages_dotnetcli.sh.template b/templates/src/csharp/build_packages_dotnetcli.sh.template
@@ -23,11 +23,17 @@
 
   # Collect the artifacts built by the previous build step
   mkdir -p nativelibs
+  # Jenkins flow (deprecated)
   cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true
+  # Kokoro flow
+  cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true
 
   # Collect protoc artifacts built by the previous build step
   mkdir -p protoc_plugins
+  # Jenkins flow (deprecated)
   cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true
+  # Kokoro flow
+  cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true
 
   dotnet restore Grpc.sln
 

diff --git a/tools/internal_ci/linux/grpc_build_packages.cfg b/tools/internal_ci/linux/grpc_build_packages.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_build_packages.sh b/tools/internal_ci/linux/grpc_build_packages.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+set -ex
+
+# Move artifacts generated by the previous step in the build chain to a place
+# where they can be accessed from within a docker container that builds
+# the packages
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f package linux -j 6
diff --git a/tools/internal_ci/linux/grpc_distribtests.cfg b/tools/internal_ci/linux/grpc_distribtests.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_distribtests.sh b/tools/internal_ci/linux/grpc_distribtests.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+set -ex
+
+# Move packages generated by the previous step in the build chain to a place
+# where they can be accessed from within a docker container that run the
+# distribtests
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f distribtest linux -j 6
diff --git a/tools/internal_ci/macos/grpc_distribtests.cfg b/tools/internal_ci/macos/grpc_distribtests.cfg
@@ -0,0 +1,27 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_distribtests.sh"
+gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/macos/grpc_distribtests.sh b/tools/internal_ci/macos/grpc_distribtests.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_macos_rc
+
+# Move packages generated by the previous step in the build chain.
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f distribtest macos
diff --git a/tools/internal_ci/windows/grpc_build_packages.bat b/tools/internal_ci/windows/grpc_build_packages.bat
@@ -0,0 +1,35 @@
+@rem Copyright 2017 gRPC authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
+@rem TODO(jtattermusch): get rid of this hack
+rename C:\Python27_32bit Python27_32bits
+rename C:\Python34_32bit Python34_32bits
+rename C:\Python35_32bit Python35_32bits
+rename C:\Python36_32bit Python36_32bits
+
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+call tools/internal_ci/helper_scripts/prepare_build_windows.bat
+
+@rem Move artifacts generated by the previous step in the build chain.
+powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts"
+dir input_artifacts
+
+python tools/run_tests/task_runner.py -f package windows -j 4 || goto :error
+goto :EOF
+
+:error
+exit /b %errorlevel%
diff --git a/tools/internal_ci/windows/grpc_build_packages.cfg b/tools/internal_ci/windows/grpc_build_packages.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}
diff --git a/tools/internal_ci/windows/grpc_distribtests.bat b/tools/internal_ci/windows/grpc_distribtests.bat
@@ -0,0 +1,35 @@
+@rem Copyright 2017 gRPC authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
+@rem TODO(jtattermusch): get rid of this hack
+rename C:\Python27_32bit Python27_32bits
+rename C:\Python34_32bit Python34_32bits
+rename C:\Python35_32bit Python35_32bits
+rename C:\Python36_32bit Python36_32bits
+
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+call tools/internal_ci/helper_scripts/prepare_build_windows.bat
+
+@rem Move packages generated by the previous step in the build chain.
+powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts"
+dir input_artifacts
+
+python tools/run_tests/task_runner.py -f distribtest windows -j 4 || goto :error
+goto :EOF
+
+:error
+exit /b %errorlevel%
diff --git a/tools/internal_ci/windows/grpc_distribtests.cfg b/tools/internal_ci/windows/grpc_distribtests.cfg
@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}