Skip to content

Commit

Permalink
Bedrock Testing V2: Parallelized testing through multi agent hosted j…
Browse files Browse the repository at this point in the history 
…obs (#566)

* Update azure-pipelines.yml for Azure Pipelines

* Init commit for synchronous Integration Tests and parallel jobs

* Added support for parallelization of Bedrock builds (#565)

* adding ssh-keygen missing from 2 integration test
  • Loading branch information
@NathanielRose
NathanielRose authored Aug 27, 2019
1 parent b0dfb1e commit 1836ff8
Show file tree
Hide file tree
Showing 2 changed files with 173 additions and 139 deletions.
287 changes: 148 additions & 139 deletions azure-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ trigger:
include:
- master
- azure-pipelines
- bedrock-testing-v2
paths:
include:
- /cluster/azure/*
Expand Down Expand Up @@ -35,151 +36,159 @@ pr:
exclude:
- '**/*.md'

variables:
GOBIN: '$(GOPATH)/bin' # Go binaries path
GOROOT: '/usr/local/go1.11' # Go installation path
GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path
modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code

jobs:
- job: Bedrock_Test_Harness
displayName: Bedrock_Build_Tests
timeoutInMinutes: 180
pool:
vmImage: 'Ubuntu-16.04'
timeoutInMinutes: 180

variables:
GOBIN: '$(GOPATH)/bin' # Go binaries path
GOROOT: '/usr/local/go1.11' # Go installation path
GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path
modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code

steps:
- checkout: self
persistCredentials: true
clean: true

- task: ShellScript@2
displayName: Validate GitOps pipeline
inputs:
scriptPath: gitops/azure-devops/build.sh
env:
VERIFY_ONLY: 1
HLD_PATH: git://github.com/Microsoft/fabrikate-production-cluster-demo.git

- script: |
mkdir -p '$(GOBIN)'
mkdir -p '$(GOPATH)/pkg'
mkdir -p '$(modulePath)'
shopt -s extglob
shopt -s dotglob
mv !(gopath) '$(modulePath)'
echo '##vso[task.prependpath]$(GOBIN)'
echo '##vso[task.prependpath]$(GOROOT)/bin'
displayName: 'Set up Go workspace'
stages:
- stage: Bedrock_Init_Validation
jobs:
- job: Bedrock_Init_Validate
steps:
- checkout: self
persistCredentials: true
clean: true
- task: ShellScript@2
displayName: Validate GitOps pipeline
inputs:
scriptPath: gitops/azure-devops/build.sh
env:
VERIFY_ONLY: 1
HLD_PATH: git://github.com/Microsoft/fabrikate-production-cluster-demo.git
- template: pipelines/bedrock-test-init.yml
- bash: |
# Fail if any of these files have warnings
find . -path ./gopath -prune -o -path ./get_helm.sh -prune -o -type f \( -name "*.sh" -o -name "*.bash" -o -name "*.ksh" \) -print |
while IFS="" read -r file
do
shellcheck "$file"
done
displayName: 'Run Shell Linting (ShellCheck)'
- script: |
terraform fmt -check=true -write=false -list=true > /dev/null
if [ $? -eq 0 ]
then
echo "Bedrock_Linting: All terraform files are valid."
else
echo "Bedrock_Linting: Terraform files invalid format, submit new commit with changes (run terraform fmt)."
exit 1
fi
gofmt -d ../bedrock/test/ | grep "^@@.*@@" > /dev/null
if [ "$?" -eq "1" ]; then
echo "Bedrock_Linting: All golang files are valid."
else
go fmt ../bedrock/test/
echo "Bedrock_Linting: Golang files invalid format, submit new commit with changes (run gofmt in /test/ directory)."
exit 1
fi
workingDirectory: '$(modulePath)/cluster'
displayName: 'Terraform lint check'
- script: |
terraform -version
wget https://releases.hashicorp.com/terraform/$(tf_version)/terraform_$(tf_version)_linux_amd64.zip -q
unzip -q terraform_$(tf_version)_linux_amd64.zip
sudo mv terraform /usr/local/bin
terraform -version
curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
chmod +x ./kubectl
sudo mv ./kubectl /usr/local/bin/kubectl
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > get_helm.sh
chmod 700 get_helm.sh
./get_helm.sh
displayName: 'Install terraform, kubectl, helm'
- stage: Bedrock_Integration_Test_Validation
jobs:
- job: Bedrock_Test_SIMPLE
displayName: Bedrock_Build_Azure_Simple
timeoutInMinutes: 180
pool:
vmImage: 'Ubuntu-16.04'
timeoutInMinutes: 180

- bash: |
# Fail if any of these files have warnings
find . -path ./gopath -prune -o -path ./get_helm.sh -prune -o -type f \( -name "*.sh" -o -name "*.bash" -o -name "*.ksh" \) -print |
while IFS="" read -r file
do
shellcheck "$file"
done
displayName: 'Run Shell Linting (ShellCheck)'
steps:
- template: pipelines/bedrock-test-init.yml
- script: |
ssh-keygen -f id_rsa -t rsa -N "bedrock@microsoft.com"
export ssh_key=$(readlink -f id_rsa.pub)
export public_key=$(cat id_rsa.pub)
find $(modulePath)/cluster/environments/azure-simple -type f -name '*.tf' -exec sed -i 's/github.com\/Microsoft\/bedrock\/cluster/..\/../g' {} \;
go test -v -run TestIT_Bedrock_AzureSimple_Test -timeout 99999s
env:
ARM_CLIENT_ID: $(ARM_CLIENT_ID)
ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID)
ARM_TENANT_ID: $(ARM_TENANT_ID)
ARM_BACKEND_STORAGE_NAME: $(ARM_BACKEND_STORAGE_NAME)
ARM_BACKEND_STORAGE_KEY: $(ARM_BACKEND_STORAGE_KEY)
ARM_BACKEND_STORAGE_CONTAINER: $(ARM_BACKEND_STORAGE_CONTAINER)
workingDirectory: '$(modulePath)/test'
displayName: 'Integration Test: Bedrock_Azure-Simple '
- script: |
terraform fmt -check=true -write=false -list=true > /dev/null
if [ $? -eq 0 ]
then
echo "Bedrock_Linting: All terraform files are valid."
else
echo "Bedrock_Linting: Terraform files invalid format, submit new commit with changes (run terraform fmt)."
exit 1
fi
gofmt -d ../bedrock/test/ | grep "^@@.*@@" > /dev/null
if [ "$?" -eq "1" ]; then
echo "Bedrock_Linting: All golang files are valid."
else
go fmt ../bedrock/test/
echo "Bedrock_Linting: Golang files invalid format, submit new commit with changes (run gofmt in /test/ directory)."
exit 1
fi
workingDirectory: '$(modulePath)/cluster'
displayName: 'Terraform lint check'
- job: Bedrock_Test_KEYVAULT
displayName: Bedrock_Build_Azure_Single_KeyVault
timeoutInMinutes: 180
pool:
vmImage: 'Ubuntu-16.04'
timeoutInMinutes: 180
variables:
GOBIN: '$(GOPATH)/bin' # Go binaries path
GOROOT: '/usr/local/go1.11' # Go installation path
GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path
modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code

- script: |
ssh-keygen -f id_rsa -t rsa -N "bedrock@microsoft.com"
export ssh_key=$(readlink -f id_rsa.pub)
export public_key=$(cat id_rsa.pub)
find $(modulePath)/cluster/environments/azure-simple -type f -name '*.tf' -exec sed -i 's/github.com\/Microsoft\/bedrock\/cluster/..\/../g' {} \;
go test -v -run TestIT_Bedrock_AzureSimple_Test -timeout 99999s
env:
ARM_CLIENT_ID: $(ARM_CLIENT_ID)
ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID)
ARM_TENANT_ID: $(ARM_TENANT_ID)
ARM_BACKEND_STORAGE_NAME: $(ARM_BACKEND_STORAGE_NAME)
ARM_BACKEND_STORAGE_KEY: $(ARM_BACKEND_STORAGE_KEY)
ARM_BACKEND_STORAGE_CONTAINER: $(ARM_BACKEND_STORAGE_CONTAINER)
workingDirectory: '$(modulePath)/test'
displayName: 'Integration Test: Bedrock_Azure-Simple '
steps:
- template: pipelines/bedrock-test-init.yml
- script: |
ssh-keygen -f id_rsa -t rsa -N "bedrock@microsoft.com"
export ssh_key=$(readlink -f id_rsa.pub)
export public_key=$(cat id_rsa.pub)
find $(modulePath)/cluster/environments/azure-single-keyvault -type f -name '*.tf' -exec sed -i 's/github.com\/Microsoft\/bedrock\/cluster/..\/../g' {} \;
go test -v -run TestIT_Bedrock_AzureCommon_KV_Test -timeout 99999s
env:
ARM_CLIENT_ID: $(ARM_CLIENT_ID)
ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID)
ARM_TENANT_ID: $(ARM_TENANT_ID)
ARM_BACKEND_STORAGE_NAME: $(ARM_BACKEND_STORAGE_NAME)
ARM_BACKEND_STORAGE_KEY: $(ARM_BACKEND_STORAGE_KEY)
ARM_BACKEND_STORAGE_CONTAINER: $(ARM_BACKEND_STORAGE_CONTAINER)
workingDirectory: '$(modulePath)/test'
displayName: 'Integration Test: Bedrock_Azure-Common-KeyVault '
- script: |
export ssh_key=$(readlink -f id_rsa.pub)
export public_key=$(cat id_rsa.pub)
find $(modulePath)/cluster/environments/azure-single-keyvault -type f -name '*.tf' -exec sed -i 's/github.com\/Microsoft\/bedrock\/cluster/..\/../g' {} \;
go test -v -run TestIT_Bedrock_AzureCommon_KV_Test -timeout 99999s
env:
ARM_CLIENT_ID: $(ARM_CLIENT_ID)
ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID)
ARM_TENANT_ID: $(ARM_TENANT_ID)
ARM_BACKEND_STORAGE_NAME: $(ARM_BACKEND_STORAGE_NAME)
ARM_BACKEND_STORAGE_KEY: $(ARM_BACKEND_STORAGE_KEY)
ARM_BACKEND_STORAGE_CONTAINER: $(ARM_BACKEND_STORAGE_CONTAINER)
workingDirectory: '$(modulePath)/test'
displayName: 'Integration Test: Bedrock_Azure-Common-KeyVault '
- job: Bedrock_Test_MULTIPLE
displayName: Bedrock_Build_Azure_Multiple_Clusters
timeoutInMinutes: 180
pool:
vmImage: 'Ubuntu-16.04'
timeoutInMinutes: 180

- script: |
export ssh_key=$(readlink -f id_rsa.pub)
export public_key=$(cat id_rsa.pub)
find $(modulePath)/cluster/environments/azure-multiple-clusters -type f -name '*.tf' -exec sed -i 's/github.com\/Microsoft\/bedrock\/cluster/..\/../g' {} \;
go test -v -run TestIT_Bedrock_AzureMC_Test -timeout 99999s
env:
ARM_CLIENT_ID: $(ARM_CLIENT_ID)
ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID)
ARM_TENANT_ID: $(ARM_TENANT_ID)
ARM_BACKEND_STORAGE_NAME: $(ARM_BACKEND_STORAGE_NAME)
ARM_BACKEND_STORAGE_KEY: $(ARM_BACKEND_STORAGE_KEY)
ARM_BACKEND_STORAGE_CONTAINER: $(ARM_BACKEND_STORAGE_CONTAINER)
workingDirectory: '$(modulePath)/test'
displayName: 'Integration Test: Bedrock_Azure-Common-MultiCluster '
variables:
GOBIN: '$(GOPATH)/bin' # Go binaries path
GOROOT: '/usr/local/go1.11' # Go installation path
GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path
modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code

# - script: |
# export ssh_key=$(readlink -f id_rsa.pub)
# export public_key=$(cat id_rsa.pub)
# find $(modulePath)/cluster/environments/azure-multiple-clusters-waf-tm-apimgmt -type f -name '*.tf' -exec sed -i 's/github.com\/Microsoft\/bedrock\/cluster/..\/../g' {} \;
# go test -v -run TestIT_Bedrock_AzureMC_APIM_Test -timeout 99999s
# env:
# ARM_CLIENT_ID: $(ARM_CLIENT_ID)
# ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
# ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID)
# ARM_TENANT_ID: $(ARM_TENANT_ID)
# ARM_BACKEND_STORAGE_NAME: $(ARM_BACKEND_STORAGE_NAME)
# ARM_BACKEND_STORAGE_KEY: $(ARM_BACKEND_STORAGE_KEY)
# ARM_BACKEND_STORAGE_CONTAINER: $(ARM_BACKEND_STORAGE_CONTAINER)
# workingDirectory: '$(modulePath)/test'
# displayName: 'Integration Test: azure-multiple-clusters-waf-tm-apimgmt '
steps:
- template: pipelines/bedrock-test-init.yml
- script: |
ssh-keygen -f id_rsa -t rsa -N "bedrock@microsoft.com"
export ssh_key=$(readlink -f id_rsa.pub)
export public_key=$(cat id_rsa.pub)
find $(modulePath)/cluster/environments/azure-multiple-clusters -type f -name '*.tf' -exec sed -i 's/github.com\/Microsoft\/bedrock\/cluster/..\/../g' {} \;
go test -v -run TestIT_Bedrock_AzureMC_Test -timeout 99999s
env:
ARM_CLIENT_ID: $(ARM_CLIENT_ID)
ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID)
ARM_TENANT_ID: $(ARM_TENANT_ID)
ARM_BACKEND_STORAGE_NAME: $(ARM_BACKEND_STORAGE_NAME)
ARM_BACKEND_STORAGE_KEY: $(ARM_BACKEND_STORAGE_KEY)
ARM_BACKEND_STORAGE_CONTAINER: $(ARM_BACKEND_STORAGE_CONTAINER)
workingDirectory: '$(modulePath)/test'
displayName: 'Integration Test: Bedrock_Azure-Common-MultiCluster '

# - script: |
# export ssh_key=$(readlink -f id_rsa.pub)
# export public_key=$(cat id_rsa.pub)
# find $(modulePath)/cluster/environments/azure-multiple-clusters-waf-tm-apimgmt -type f -name '*.tf' -exec sed -i 's/github.com\/Microsoft\/bedrock\/cluster/..\/../g' {} \;
# go test -v -run TestIT_Bedrock_AzureMC_APIM_Test -timeout 99999s
# env:
# ARM_CLIENT_ID: $(ARM_CLIENT_ID)
# ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
# ARM_SUBSCRIPTION_ID: $(ARM_SUBSCRIPTION_ID)
# ARM_TENANT_ID: $(ARM_TENANT_ID)
# ARM_BACKEND_STORAGE_NAME: $(ARM_BACKEND_STORAGE_NAME)
# ARM_BACKEND_STORAGE_KEY: $(ARM_BACKEND_STORAGE_KEY)
# ARM_BACKEND_STORAGE_CONTAINER: $(ARM_BACKEND_STORAGE_CONTAINER)
# workingDirectory: '$(modulePath)/test'
# displayName: 'Integration Test: azure-multiple-clusters-waf-tm-apimgmt '
25 changes: 25 additions & 0 deletions pipelines/bedrock-test-init.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
steps:
- script: |
mkdir -p '$(GOBIN)'
mkdir -p '$(GOPATH)/pkg'
mkdir -p '$(modulePath)'
shopt -s extglob
shopt -s dotglob
mv !(gopath) '$(modulePath)'
echo '##vso[task.prependpath]$(GOBIN)'
echo '##vso[task.prependpath]$(GOROOT)/bin'
displayName: 'Set up Go workspace'

- script: |
terraform -version
wget https://releases.hashicorp.com/terraform/$(tf_version)/terraform_$(tf_version)_linux_amd64.zip -q
unzip -q terraform_$(tf_version)_linux_amd64.zip
sudo mv terraform /usr/local/bin
terraform -version
curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
chmod +x ./kubectl
sudo mv ./kubectl /usr/local/bin/kubectl
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > get_helm.sh
chmod 700 get_helm.sh
./get_helm.sh
displayName: 'Install terraform, kubectl, helm'

0 comments on commit 1836ff8

Please sign in to comment.