gha: collect kafka data in parallel

Issue: ZENKO-4787
scality · Mar 29, 2024 · f7631b7 · f7631b7
1 parent c86f191
commit f7631b7
Showing 1 changed file with 17 additions and 18 deletions.
diff --git a/.github/actions/archive-artifacts/action.yaml b/.github/actions/archive-artifacts/action.yaml
@@ -17,26 +17,25 @@ runs:
     - name: Dump kafka
       shell: bash
       run: |-
-        set -exu;
+        set -exu
 
         NAMESPACE=${NAMESPACE:-default}
-        KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME:-end2end}-base-queue -o jsonpath='{.items[0].metadata.name}');
+        KAFKA=$(kubectl get pods -n ${NAMESPACE} -lkafka_cr=${ZENKO_NAME:-end2end}-base-queue -o jsonpath='{.items[0].metadata.name}')
 
-        kubectl exec -itn ${NAMESPACE} ${KAFKA} -c kafka -- \
+        kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \
             env KAFKA_OPTS= kafka-topics.sh --bootstrap-server :9092 --list \
-            > /tmp/artifacts/data/${STAGE}/kafka-topics.log;
-        kubectl exec -itn ${NAMESPACE} ${KAFKA} -c kafka -- \
+            > /tmp/artifacts/data/${STAGE}/kafka-topics.log
+
+        kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \
             env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --list \
-            | grep -v '^__' \
-            > /tmp/artifacts/data/${STAGE}/kafka-consumer-groups.log;
-        for topic in $(cat /tmp/artifacts/data/${STAGE}/kafka-topics.log); do
-            kubectl exec -itn ${NAMESPACE} ${KAFKA} -c kafka -- \
-                env KAFKA_OPTS= kafka-console-consumer.sh --bootstrap-server :9092 --topic $topic \
-                                                          --from-beginning --timeout-ms 1000 \
-                > /tmp/artifacts/data/${STAGE}/kafka-messages-$topic.log;
-        done;
-        for cg in $(cat /tmp/artifacts/data/${STAGE}/kafka-consumer-groups.log); do
-            kubectl exec -itn ${NAMESPACE} ${KAFKA} -c kafka  -- \
-                env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --describe --group $cg \
-                > /tmp/artifacts/data/${STAGE}/kafka-offsets-$cg.log;
-        done;
+            > /tmp/artifacts/data/${STAGE}/kafka-consumer-groups.log
+
+        kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka  -- \
+                   env KAFKA_OPTS= kafka-consumer-groups.sh --bootstrap-server :9092 --describe --all-groups \
+                   > /tmp/artifacts/data/${STAGE}/kafka-offsets.log
+
+        cat /tmp/artifacts/data/${STAGE}/kafka-topics.log | xargs -P 15 -I {} \
+            sh -c "kubectl exec -in ${NAMESPACE} ${KAFKA} -c kafka -- \
+                   env KAFKA_OPTS= kafka-console-consumer.sh --bootstrap-server :9092 --topic {} \
+                                                             --from-beginning --timeout-ms 60000 \
+                   > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log"