Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Node-5583 Required variables for smoke-tests #175

Merged
merged 2 commits into from
Aug 9, 2024
Merged

Node-5583 Required variables for smoke-tests #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
234ec88
NODE-5583 add mandatory function
braek-neck Aug 6, 2024
80e4272
NODE-5583 add dockerhub auth
braek-neck Aug 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,22 @@ jobs:
- name: Checkout
uses: actions/checkout@v4

- name: Import secrets
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e # v2.5.0
id: secrets
with:
exportEnv: true
url: ${{ secrets.VAULT_URL }}
role: ${{ secrets.VAULT_ROLE }}
method: kubernetes
path: kubernetes-ci
secrets: |
kv-gitlab-ci/data/github/shared/dockerhub-creds user | DOCKERHUB_USER ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password | DOCKERHUB_PASSWORD ;

- name: Build
run: |
echo ${DOCKERHUB_PASSWORD} | docker login -u ${DOCKERHUB_USER} --password-stdin
IMAGE="wallarm/sidecar-controller"
make build TAG=${GITHUB_SHA} IMAGE=${IMAGE} PLATFORMS=linux/${{ matrix.ARCH }} BUILDX_ARGS=--load
docker save -o sidecar-${{ matrix.ARCH }}.tar ${IMAGE}:${GITHUB_SHA}
Expand Down Expand Up @@ -117,6 +131,8 @@ jobs:
kv-gitlab-ci/data/github/${{ env.env_code }} allure_server_token ;
kv-gitlab-ci/data/github/shared/smoke-tests-registry-creds token_name ;
kv-gitlab-ci/data/github/shared/smoke-tests-registry-creds token_secret ;
kv-gitlab-ci/data/github/shared/dockerhub-creds user ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password ;

- name: Checkout
uses: actions/checkout@v4
Expand All @@ -136,6 +152,8 @@ jobs:
env:
SKIP_CLUSTER_CREATION: true
SKIP_IMAGE_CREATION: true
DOCKERHUB_USER: ${{ steps.secrets.outputs.user }}
DOCKERHUB_PASSWORD: ${{ steps.secrets.outputs.password }}
INJECTION_STRATEGY: ${{ matrix.INJECTION_STRATEGY }}
WALLARM_API_TOKEN: ${{ steps.secrets.outputs.api_token }}
WALLARM_API_HOST: ${{ steps.secrets.outputs.api_host }}
Expand All @@ -145,6 +163,7 @@ jobs:
SMOKE_REGISTRY_SECRET: ${{ steps.secrets.outputs.token_secret }}
WEBHOOK_API_KEY: ${{ steps.secrets.outputs.webhook_api_key }}
WEBHOOK_UUID: ${{ steps.secrets.outputs.webhook_uuid }}
ALLURE_PROJECT_ID: 10
ALLURE_UPLOAD_REPORT: true
ALLURE_GENERATE_REPORT: true
ALLURE_TOKEN: ${{ steps.secrets.outputs.allure_server_token }}
Expand Down
16 changes: 13 additions & 3 deletions example-env.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@
### the main docker containers via docker-compose.
### Please rename this file to "env.ini" for makeing
### it work.
###
WALLARM_API_TOKEN=xxxxx-base64-base64-base64-base64-xxxxx
WALLARM_API_HOST=api.wallarm.com
# Wallarm API settings
WALLARM_API_TOKEN=...
WALLARM_API_HOST=us1.api.wallarm.com
WALLARM_API_PRESET=us1
CLIENT_ID=...
USER_TOKEN=...
SMOKE_REGISTRY_TOKEN=...
SMOKE_REGISTRY_SECRET=glpat-...

WEBHOOK_API_KEY=...
WEBHOOK_UUID=...

PYTEST_WORKERS=10
167 changes: 167 additions & 0 deletions test/smoke/functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
# This file used for import in other files

RED='\033[0;31m'
NC='\033[0m'

function check_mandatory_vars() {

declare -a mandatory
declare -a allure_mandatory

mandatory=(
WALLARM_API_TOKEN
WALLARM_API_HOST
WALLARM_API_PRESET
CLIENT_ID
USER_TOKEN
WEBHOOK_API_KEY
WEBHOOK_UUID
SMOKE_REGISTRY_TOKEN
SMOKE_REGISTRY_SECRET
)

env_list=""

for var in "${mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
env_list+=" $var"
fi
done

if [[ "${ALLURE_UPLOAD_REPORT:-false}" == "true" ]]; then
allure_mandatory=(
ALLURE_TOKEN
ALLURE_ENVIRONMENT_ARCH
ALLURE_PROJECT_ID
ALLURE_GENERATE_REPORT
ALLURE_ENVIRONMENT_K8S
)

for var in "${allure_mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
env_list+=" $var"
fi
done
fi

if [[ -n "$env_list" ]]; then
for var in ${env_list}; do
echo -e "${RED}Environment variable $var must be set${NC}"
done
exit 1
fi

}

function cleanup() {
if [[ "${KUBETEST_IN_DOCKER:-}" == "true" ]]; then
kind "export" logs --name ${KIND_CLUSTER_NAME} "${ARTIFACTS}/logs" || true
fi
if [[ "${CI:-}" == "true" ]]; then
kind delete cluster \
--verbosity=${KIND_LOG_LEVEL} \
--name ${KIND_CLUSTER_NAME}
fi
}

function get_controller_logs_and_fail() {
echo "#################################"
echo "######## Controller logs ########"
echo "#################################"
kubectl logs -l "app.kubernetes.io/component=controller" --tail=-1 || true
echo -e "#################################\n"

echo "#####################################"
echo "######## Post-analytics logs ########"
echo -e "#####################################\n"
for CONTAINER in antibot appstructure supervisord tarantool ; do
echo "#######################################"
echo "###### ${CONTAINER} container logs ######"
echo -e "#######################################\n"
kubectl logs -l "app.kubernetes.io/component=postanalytics" -c ${CONTAINER} --tail=-1 || true
echo -e "#######################################\n"
done

for COMPONENT in controller postanalytics ; do
echo "#######################################"
echo "###### Describe ${COMPONENT} pod ######"
echo -e "#######################################\n"
kubectl describe po -l "app.kubernetes.io/component=${COMPONENT}"
echo -e "#######################################\n"
done

exit 1
}

###

function get_logs_and_fail() {
get_logs
extra_debug_logs
clean_allure_report
exit 1
}

function get_logs() {
echo "#################################"
echo "######## Controller logs ########"
echo "#################################"
kubectl logs -l "app.kubernetes.io/component=controller" --tail=-1
echo -e "#################################\n"

echo "#################################"
echo "######## Post-analytics Pod #####"
echo "#################################"
for CONTAINER in antibot appstructure supervisord tarantool ; do
echo "#######################################"
echo "###### ${CONTAINER} container logs ######"
echo -e "#######################################\n"
kubectl logs -l "app.kubernetes.io/component=postanalytics" -c ${CONTAINER} --tail=-1
echo -e "#######################################\n"
done

echo "#################################"
echo "######## Application Pod ########"
echo -e "#################################\n"

echo "####################################################"
echo "###### List directory /opt/wallarm/etc/wallarm #####"
echo "####################################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/etc/wallarm && cat /opt/wallarm/etc/wallarm/node.yaml" || true
echo -e "#####################################################\n"

echo "############################################"
echo "###### List directory /var/lib/nginx/wallarm"
echo "############################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/var/lib/nginx/wallarm && ls -laht /opt/wallarm/var/lib/nginx/wallarm/shm" || true
echo -e "############################################\n"

echo "############################################################"
echo "###### List directory /opt/wallarm/var/lib/wallarm-acl #####"
echo "############################################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/var/lib/wallarm-acl" || true
echo -e "############################################################\n"

echo "#################################"
echo "######## Application Pod Logs ###"
echo -e "#################################\n"
kubectl logs -l "wallarm-sidecar=enabled" --all-containers --ignore-errors --since=1h
echo -e "#################################\n"
}

function extra_debug_logs {
echo "############################################"
echo "###### Extra cluster debug info ############"
echo "############################################"

echo "Grepping cluster OOMKilled events..."
kubectl get events -A | grep -i OOMKill || true

echo "Displaying pods state in default namespace..."
kubectl get pods

}

function clean_allure_report() {
[[ "$ALLURE_GENERATE_REPORT" == false && -d "allure_report" ]] && rm -rf allure_report/* 2>/dev/null || true
}
102 changes: 7 additions & 95 deletions test/smoke/run-smoke-suite.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ set -o errexit
set -o nounset
set -o pipefail

# check if all mandatory vars was defined
source "${PWD}/test/smoke/functions.sh"
check_mandatory_vars

export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ingress-smoke-test}
export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"

Expand All @@ -20,13 +24,13 @@ SMOKE_IMAGE_TAG="${SMOKE_IMAGE_TAG:-latest}"

# Allure related variables
ALLURE_ENDPOINT="${ALLURE_ENDPOINT:-https://allure.wallarm.com}"
ALLURE_PROJECT_ID=${ALLURE_PROJECT_ID:-10}
ALLURE_PROJECT_ID=${ALLURE_PROJECT_ID}
ALLURE_RESULTS="${ALLURE_RESULTS:-/tests/_out/allure_report}"
ALLURE_UPLOAD_REPORT="${ALLURE_UPLOAD_REPORT:-false}"
ALLURE_GENERATE_REPORT="${ALLURE_GENERATE_REPORT:-false}"

# Pytest related variables
CLIENT_ID="${CLIENT_ID:-5}"
CLIENT_ID="${CLIENT_ID}"
WALLARM_API_CA_VERIFY="${WALLARM_API_CA_VERIFY:-true}"
WALLARM_API_HOST="${WALLARM_API_HOST:-api.wallarm.com}"
WALLARM_API_PRESET="${WALLARM_API_PRESET:-eu1}"
Expand All @@ -36,98 +40,6 @@ PYTEST_WORKERS="${PYTEST_WORKERS:-15}"
#TODO We need it here just to don't let test fail. Remove this variable when test will be fixed.
HOSTNAME_OLD_NODE="smoke-tests-old-node"

function clean_allure_report() {
[[ "$ALLURE_GENERATE_REPORT" == false && -d "allure_report" ]] && rm -rf allure_report/* 2>/dev/null || true
}

function get_logs_and_fail() {
get_logs
extra_debug_logs
clean_allure_report
exit 1
}

function get_logs() {
echo "#################################"
echo "######## Controller logs ########"
echo "#################################"
kubectl logs -l "app.kubernetes.io/component=controller" --tail=-1
echo -e "#################################\n"

echo "#################################"
echo "######## Post-analytics Pod #####"
echo "#################################"
for CONTAINER in appstructure supervisord tarantool ; do
echo "#######################################"
echo "###### ${CONTAINER} container logs ######"
echo -e "#######################################\n"
kubectl logs -l "app.kubernetes.io/component=postanalytics" -c ${CONTAINER} --tail=-1
echo -e "#######################################\n"
done

echo "#################################"
echo "######## Application Pod ########"
echo -e "#################################\n"

echo "####################################################"
echo "###### List directory /opt/wallarm/etc/wallarm #####"
echo "####################################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/etc/wallarm && cat /opt/wallarm/etc/wallarm/node.yaml" || true
echo -e "#####################################################\n"

echo "############################################"
echo "###### List directory /var/lib/nginx/wallarm"
echo "############################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/var/lib/nginx/wallarm && ls -laht /opt/wallarm/var/lib/nginx/wallarm/shm" || true
echo -e "############################################\n"

echo "############################################################"
echo "###### List directory /opt/wallarm/var/lib/wallarm-acl #####"
echo "############################################################"
kubectl exec "${POD}" -c sidecar-proxy -- sh -c "ls -laht /opt/wallarm/var/lib/wallarm-acl" || true
echo -e "############################################################\n"

echo "#################################"
echo "######## Application Pod Logs ###"
echo -e "#################################\n"
kubectl logs -l "wallarm-sidecar=enabled" --all-containers --ignore-errors --since=1h
echo -e "#################################\n"
}


function extra_debug_logs {
echo "############################################"
echo "###### Extra cluster debug info ############"
echo "############################################"

echo "Grepping cluster OOMKilled events..."
kubectl get events -A | grep -i OOMKill || true

echo "Displaying pods state in default namespace..."
kubectl get pods

}

declare -a mandatory
mandatory=(
CLIENT_ID
USER_TOKEN
SMOKE_REGISTRY_TOKEN
SMOKE_REGISTRY_SECRET
)

missing=false
for var in "${mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
echo "Environment variable $var must be set"
missing=true
fi
done

if [ "$missing" = true ]; then
exit 1
fi

if [[ "${CI:-false}" == "false" ]]; then
trap 'kubectl delete pod pytest --now --ignore-not-found' EXIT ERR
# Colorize pytest output if run locally
Expand Down Expand Up @@ -229,4 +141,4 @@ EXEC_CMD="env $GITHUB_VARS $RUN_TESTS -n ${PYTEST_WORKERS} ${PYTEST_ARGS}"
# shellcheck disable=SC2086
kubectl exec pytest ${EXEC_ARGS} -- ${EXEC_CMD} || get_logs_and_fail
extra_debug_logs
clean_allure_report
clean_allure_report
Loading
Loading