Fix load test scenarios, runner, and CI threshold checks

- Add constant-arrival-rate scenarios to all 4 service scripts (api,
  darkwatch, spamshield, voiceprint) to enforce 500 req/s target
- Fix defaultThresholds() to return { thresholds: {...} } so
  http_req_duration and errors thresholds are actually applied
- Rewrite run-all.sh: per-service summary files, proper env var
  passing (DURATION, API_TOKEN), fixed threshold aggregation
- Update CI workflow threshold check jq to match new threshold-results
  structure (.services.<name>.exitCode)

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/ci.yml

@@ -217,14 +217,13 @@ jobs:
         if: always()
         run: |
           if [ -f scripts/load-test/reports/threshold-results.json ]; then
-            FAILURES=$(jq -r '.metrics | to_entries[] | select(.value.passes == false) | .key' scripts/load-test/reports/threshold-results.json)
+            FAILURES=$(jq -r '[.services | to_entries[] | select(.value.exitCode != 0) | .key] | join(", ")' scripts/load-test/reports/threshold-results.json 2>/dev/null || echo "")
-            if [ -n "$FAILURES" ]; then
+            if [ -n "$FAILURES" ] && [ "$FAILURES" != "" ]; then
-              echo "❌ P99 threshold failures:"
+              echo "❌ Load test failures: $FAILURES"
-              echo "$FAILURES"
               exit 1
             else
-              echo "✅ All P99 thresholds passed"
+              echo "✅ All load tests passed"
             fi
           else
-            echo "No threshold results file found"
+            echo "⚠️ No threshold results file found"
           fi

.github/workflows/load-test.yml

@@ -64,14 +64,13 @@ jobs:
         if: always()
         run: |
           if [ -f scripts/load-test/reports/threshold-results.json ]; then
-            FAILURES=$(jq -r '.metrics | to_entries[] | select(.value.passes == false) | .key' scripts/load-test/reports/threshold-results.json 2>/dev/null || echo "")
+            FAILURES=$(jq -r '[.services | to_entries[] | select(.value.exitCode != 0) | .key] | join(", ")' scripts/load-test/reports/threshold-results.json 2>/dev/null || echo "")
-            if [ -n "$FAILURES" ]; then
+            if [ -n "$FAILURES" ] && [ "$FAILURES" != "" ]; then
-              echo "❌ P99 threshold failures:"
+              echo "❌ Load test failures: $FAILURES"
-              echo "$FAILURES"
               exit 1
             else
-              echo "✅ All P99 thresholds passed"
+              echo "✅ All load tests passed"
             fi
           else
-            echo "No threshold results file found"
+            echo "⚠️ No threshold results file found"
           fi

scripts/load-test/lib/common.js

@@ -16,8 +16,10 @@ export function getDuration() {
 
 export function defaultThresholds(p99ms) {
   return {
-    http_req_duration: [`p(99)<${p99ms}`],
+    thresholds: {
-    errors: ['rate<0.01'],
+      http_req_duration: [`p(99)<${p99ms}`],
+      errors: ['rate<0.01'],
+    },
   };
 }
 

scripts/load-test/run-all.sh

@@ -8,7 +8,6 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPORT_DIR="${SCRIPT_DIR}/reports"
 TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
-SUMMARY_FILE="${REPORT_DIR}/summary-${TIMESTAMP}.json"
 THRESHOLD_FILE="${REPORT_DIR}/threshold-results.json"
 SERVICE="${1:-all}"
 
@@ -17,6 +16,7 @@ mkdir -p "$REPORT_DIR"
 BASE_URL="${LOAD_TEST_BASE_URL:-http://localhost:3000}"
 TARGET_RPS="${TARGET_RPS:-500}"
 DURATION="${DURATION:-300s}"
+API_TOKEN="${API_TOKEN:-test-token}"
 
 echo "=== ShieldAI Combined Load Test ==="
 echo "Timestamp: $TIMESTAMP"
@@ -26,7 +26,7 @@ echo "Duration: $DURATION"
 echo "Service: $SERVICE"
 echo ""
 
-K6_OPTS="--duration $DURATION --summary-export $SUMMARY_FILE"
+K6_OPTS="--summary-export ${REPORT_DIR}/summary-${TIMESTAMP}.json"
 
 if [[ -n "${K6_CLOUD_TOKEN:-}" ]]; then
   K6_OPTS="$K6_OPTS --out cloud"
@@ -35,17 +35,23 @@ fi
 
 declare -A EXIT_CODES
 ALL_PASSED=true
+SERVICE_ENV="BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS DURATION=$DURATION API_TOKEN=$API_TOKEN"
 
 run_service_test() {
   local name=$1
   local script=$2
-  local env_vars=$3
+  local summary_file="${REPORT_DIR}/${name}-summary-${TIMESTAMP}.json"
 
   echo ""
   echo "=== Running $name Load Test ==="
 
+  local opts="--summary-export $summary_file"
+  if [[ -n "${K6_CLOUD_TOKEN:-}" ]]; then
+    opts="$opts --out cloud"
+  fi
+
   set +e
-  eval "$env_vars" k6 run $K6_OPTS "$script"
+  eval "$SERVICE_ENV" k6 run $opts "$script"
   EXIT_CODE=$?
   set -e
 
@@ -59,33 +65,49 @@ run_service_test() {
 }
 
 if [[ "$SERVICE" == "all" || "$SERVICE" == "api" ]]; then
-  run_service_test "api" "${SCRIPT_DIR}/services/api.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS"
+  run_service_test "api" "${SCRIPT_DIR}/services/api.js"
 fi
 
 if [[ "$SERVICE" == "all" || "$SERVICE" == "darkwatch" ]]; then
-  run_service_test "darkwatch" "${SCRIPT_DIR}/services/darkwatch.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS"
+  run_service_test "darkwatch" "${SCRIPT_DIR}/services/darkwatch.js"
 fi
 
 if [[ "$SERVICE" == "all" || "$SERVICE" == "spamshield" ]]; then
-  run_service_test "spamshield" "${SCRIPT_DIR}/services/spamshield.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS"
+  run_service_test "spamshield" "${SCRIPT_DIR}/services/spamshield.js"
 fi
 
 if [[ "$SERVICE" == "all" || "$SERVICE" == "voiceprint" ]]; then
-  run_service_test "voiceprint" "${SCRIPT_DIR}/services/voiceprint.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS"
+  run_service_test "voiceprint" "${SCRIPT_DIR}/services/voiceprint.js"
 fi
 
+# Aggregate threshold results from all service summaries
 echo ""
 echo "=== Load Test Results ==="
-echo "Summary saved to: $SUMMARY_FILE"
 
-# Aggregate thresholds
+# Build threshold-results.json from k6 summary exports
-jq -n \
+jq -n --arg timestamp "$TIMESTAMP" --arg base_url "$BASE_URL" --arg target_rps "$TARGET_RPS" \
-  --arg timestamp "$TIMESTAMP" \
+  '{timestamp: $timestamp, base_url: $base_url, target_rps: $target_rps, services: {}}' \
-  --arg base_url "$BASE_URL" \
+  > "$THRESHOLD_FILE"
-  --arg target_rps "$TARGET_RPS" \
+
-  --argjson exit_codes "$(printf '%s\n' "${!EXIT_CODES[@]}" | jq -R . | jq -s 'with_entries(.key = .[0])' | while read key; do echo; done)" \
+for name in "${!EXIT_CODES[@]}"; do
-  '{timestamp: $timestamp, base_url: $base_url, target_rps: $target_rps, services: $exit_codes}' \
+  summary_file="${REPORT_DIR}/${name}-summary-${TIMESTAMP}.json"
-  > "$THRESHOLD_FILE" 2>/dev/null || true
+  if [[ -f "$summary_file" ]]; then
+    jq --arg name "$name" --argjson exit_code "${EXIT_CODES[$name]}" \
+      '.services[$name] = {
+        exitCode: $exit_code,
+        passed: ($exit_code == 0),
+        metrics: (input | .metrics // {})
+      }' \
+      "$THRESHOLD_FILE" "$summary_file" \
+      > "${THRESHOLD_FILE}.tmp" && mv "${THRESHOLD_FILE}.tmp" "$THRESHOLD_FILE"
+  else
+    jq --arg name "$name" --argjson exit_code "${EXIT_CODES[$name]}" \
+      '.services[$name] = {exitCode: $exit_code, passed: ($exit_code == 0)}' \
+      "$THRESHOLD_FILE" > "${THRESHOLD_FILE}.tmp" && mv "${THRESHOLD_FILE}.tmp" "$THRESHOLD_FILE"
+  fi
+done
+
+echo "Threshold results saved to: $THRESHOLD_FILE"
 
 for service in "${!EXIT_CODES[@]}"; do
   status="pass"

scripts/load-test/services/api.js

@@ -7,7 +7,22 @@ const errorRate = new Rate('errors');
 const notificationLatency = new Trend('notification_p99');
 const correlationLatency = new Trend('correlation_p99');
 
+const TARGET_RPS = getTargetRps();
+const DURATION = getDuration();
+
 export const options = {
+  scenarios: {
+    sustained_load: {
+      executor: 'constant-arrival-rate',
+      duration: DURATION,
+      rate: TARGET_RPS,
+      preAllocatedVUs: 20,
+      maxVUs: 100,
+      startTime: '0s',
+      exec: 'default',
+      tags: { scenario: 'sustained_load' },
+    },
+  },
   thresholds: {
     ...defaultThresholds(250).thresholds,
     notification_p99: ['p(99)<500'],

scripts/load-test/services/darkwatch.js

@@ -8,7 +8,22 @@ const scanLatency = new Trend('scan_p99');
 const watchlistLatency = new Trend('watchlist_p99');
 const alertLatency = new Trend('alert_p99');
 
+const TARGET_RPS = getTargetRps();
+const DURATION = getDuration();
+
 export const options = {
+  scenarios: {
+    sustained_load: {
+      executor: 'constant-arrival-rate',
+      duration: DURATION,
+      rate: TARGET_RPS,
+      preAllocatedVUs: 20,
+      maxVUs: 100,
+      startTime: '0s',
+      exec: 'default',
+      tags: { scenario: 'sustained_load' },
+    },
+  },
   thresholds: {
     ...defaultThresholds(200).thresholds,
     scan_p99: ['p(99)<300'],

scripts/load-test/services/spamshield.js

@@ -8,7 +8,22 @@ const smsClassifyP99 = new Trend('sms_classify_p99');
 const numberReputationP99 = new Trend('number_reputation_p99');
 const callAnalyzeP99 = new Trend('call_analyze_p99');
 
+const TARGET_RPS = getTargetRps();
+const DURATION = getDuration();
+
 export const options = {
+  scenarios: {
+    sustained_load: {
+      executor: 'constant-arrival-rate',
+      duration: DURATION,
+      rate: TARGET_RPS,
+      preAllocatedVUs: 20,
+      maxVUs: 100,
+      startTime: '0s',
+      exec: 'default',
+      tags: { scenario: 'sustained_load' },
+    },
+  },
   thresholds: {
     ...defaultThresholds(400).thresholds,
     sms_classify_p99: ['p(99)<150'],

scripts/load-test/services/voiceprint.js

@@ -8,7 +8,22 @@ const enrollmentLatency = new Trend('enrollment_p99');
 const verificationLatency = new Trend('verification_p99');
 const modelLatency = new Trend('model_retrieval_p99');
 
+const TARGET_RPS = getTargetRps();
+const DURATION = getDuration();
+
 export const options = {
+  scenarios: {
+    sustained_load: {
+      executor: 'constant-arrival-rate',
+      duration: DURATION,
+      rate: TARGET_RPS,
+      preAllocatedVUs: 20,
+      maxVUs: 100,
+      startTime: '0s',
+      exec: 'default',
+      tags: { scenario: 'sustained_load' },
+    },
+  },
   thresholds: {
     ...defaultThresholds(250).thresholds,
     enrollment_p99: ['p(99)<500'],