Add load testing job to GitHub Actions CI pipeline (FRE-4931)

- Add load-test job to ci.yml that runs after docker-build on push to main
- Create combined load test runner (scripts/load-test/run-all.sh) for all services
- Create k6 load test scripts for api, darkwatch, spamshield, and voiceprint
- Add shared k6 utilities (lib/common.js)
- Update load-test.yml to support all services and report artifacts
- Configure k6 cloud output and P99 threshold validation
- Generate load test report as CI artifact

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/ci.yml

@@ -177,3 +177,54 @@ jobs:
         env:
           TF_VAR_hibp_api_key: ${{ secrets.HIBP_API_KEY }}
           TF_VAR_resend_api_key: ${{ secrets.RESEND_API_KEY }}
+
+  load-test:
+    name: Load Test
+    runs-on: ubuntu-latest
+    needs: [lint, typecheck, test, docker-build]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    environment: staging
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install k6
+        run: |
+          curl -s https://github.com/grafana/k6/releases/download/v0.50.0/k6-linux-amd64.tar.gz -L | tar xz
+          sudo mv k6 /usr/local/bin/
+          k6 version
+
+      - name: Run combined load tests
+        run: |
+          chmod +x scripts/load-test/run-all.sh
+          ./scripts/load-test/run-all.sh
+        env:
+          LOAD_TEST_BASE_URL: ${{ secrets.LOAD_TEST_BASE_URL || 'http://localhost:3000' }}
+          API_TOKEN: ${{ secrets.LOAD_TEST_API_TOKEN || 'test-token' }}
+          TARGET_RPS: ${{ vars.LOAD_TEST_TARGET_RPS || '500' }}
+          DURATION: ${{ vars.LOAD_TEST_DURATION || '300s' }}
+          K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN || '' }}
+          K6_CLOUD_PROJECT_ID: ${{ vars.K6_CLOUD_PROJECT_ID || '' }}
+
+      - name: Upload load test report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: load-test-report-${{ github.sha }}
+          path: scripts/load-test/reports/
+          retention-days: 30
+
+      - name: Check P99 thresholds
+        if: always()
+        run: |
+          if [ -f scripts/load-test/reports/threshold-results.json ]; then
+            FAILURES=$(jq -r '.metrics | to_entries[] | select(.value.passes == false) | .key' scripts/load-test/reports/threshold-results.json)
+            if [ -n "$FAILURES" ]; then
+              echo "❌ P99 threshold failures:"
+              echo "$FAILURES"
+              exit 1
+            else
+              echo "✅ All P99 thresholds passed"
+            fi
+          else
+            echo "No threshold results file found"
+          fi

.github/workflows/load-test.yml

@@ -13,16 +13,24 @@ on:
         description: 'Test duration'
         required: false
         default: '300s'
+      service:
+        description: 'Service to test (all, api, darkwatch, spamshield, voiceprint)'
+        required: false
+        default: 'all'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  NODE_VERSION: "20"
+
 jobs:
-  load-test-voiceprint:
+  load-test:
-    name: Voiceprint Load Test
+    name: Load Test (${{ github.event.inputs.service || 'all' }})
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
+    environment: staging
     steps:
       - uses: actions/checkout@v4
 
@@ -32,23 +40,38 @@ jobs:
           sudo mv k6 /usr/local/bin/
           k6 version
 
-      - name: Run Voiceprint load tests
+      - name: Run load tests
         run: |
-          cd load-tests/voiceprint
+          chmod +x scripts/load-test/run-all.sh
-          ./run.sh mixed
+          ./scripts/load-test/run-all.sh ${{ github.event.inputs.service || 'all' }}
         env:
-          VOICEPRINT_BASE_URL: ${{ secrets.LOAD_TEST_BASE_URL || 'http://localhost:3000' }}
+          LOAD_TEST_BASE_URL: ${{ secrets.LOAD_TEST_BASE_URL || 'http://localhost:3000' }}
           API_TOKEN: ${{ secrets.LOAD_TEST_API_TOKEN || 'test-token' }}
           TARGET_RPS: ${{ github.event.inputs.target_rps || '500' }}
           DURATION: ${{ github.event.inputs.duration || '300s' }}
-          ENROLLMENT_P99_MS: '500'
+          K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN || '' }}
-          VERIFICATION_P99_MS: '250'
+          K6_CLOUD_PROJECT_ID: ${{ vars.K6_CLOUD_PROJECT_ID || '' }}
-          MODEL_RETRIEVAL_P99_MS: '100'
 
-      - name: Upload results
+      - name: Upload load test report
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: load-test-results-${{ github.sha }}
+          name: load-test-report-${{ github.sha }}
-          path: load-tests/voiceprint/results/
+          path: scripts/load-test/reports/
           retention-days: 30
+
+      - name: Check P99 thresholds
+        if: always()
+        run: |
+          if [ -f scripts/load-test/reports/threshold-results.json ]; then
+            FAILURES=$(jq -r '.metrics | to_entries[] | select(.value.passes == false) | .key' scripts/load-test/reports/threshold-results.json 2>/dev/null || echo "")
+            if [ -n "$FAILURES" ]; then
+              echo "❌ P99 threshold failures:"
+              echo "$FAILURES"
+              exit 1
+            else
+              echo "✅ All P99 thresholds passed"
+            fi
+          else
+            echo "No threshold results file found"
+          fi

scripts/load-test/lib/common.js

@@ -0,0 +1,42 @@
+import { Rate, Trend } from 'k6/metrics';
+
+export const errorRate = new Rate('errors');
+
+export function getBaseUrl() {
+  return __ENV.BASE_URL || 'http://localhost:3000';
+}
+
+export function getTargetRps() {
+  return parseInt(__ENV.TARGET_RPS || '500', 10);
+}
+
+export function getDuration() {
+  return __ENV.DURATION || '300s';
+}
+
+export function defaultThresholds(p99ms) {
+  return {
+    http_req_duration: [`p(99)<${p99ms}`],
+    errors: ['rate<0.01'],
+  };
+}
+
+export function checkResponse(res, expectedStatus = 200) {
+  const pass = check(res, {
+    'status is expected': (r) => r.status === expectedStatus,
+    'response time OK': (r) => r.timings.duration < 5000,
+  });
+  if (!pass) {
+    errorRate.add(1);
+  }
+  return pass;
+}
+
+export function randomString(length = 10) {
+  const chars = 'abcdefghijklmnopqrstuvwxyz0123456789';
+  let result = '';
+  for (let i = 0; i < length; i++) {
+    result += chars.charAt(Math.floor(Math.random() * chars.length));
+  }
+  return result;
+}

scripts/load-test/run-all.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+# Combined load test runner for all ShieldAI services
+# Usage: ./run-all.sh [service]
+#   service: all (default), api, darkwatch, spamshield, voiceprint
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPORT_DIR="${SCRIPT_DIR}/reports"
+TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
+SUMMARY_FILE="${REPORT_DIR}/summary-${TIMESTAMP}.json"
+THRESHOLD_FILE="${REPORT_DIR}/threshold-results.json"
+SERVICE="${1:-all}"
+
+mkdir -p "$REPORT_DIR"
+
+BASE_URL="${LOAD_TEST_BASE_URL:-http://localhost:3000}"
+TARGET_RPS="${TARGET_RPS:-500}"
+DURATION="${DURATION:-300s}"
+
+echo "=== ShieldAI Combined Load Test ==="
+echo "Timestamp: $TIMESTAMP"
+echo "Base URL: $BASE_URL"
+echo "Target RPS: $TARGET_RPS"
+echo "Duration: $DURATION"
+echo "Service: $SERVICE"
+echo ""
+
+K6_OPTS="--duration $DURATION --summary-export $SUMMARY_FILE"
+
+if [[ -n "${K6_CLOUD_TOKEN:-}" ]]; then
+  K6_OPTS="$K6_OPTS --out cloud"
+  echo "k6 cloud output: enabled"
+fi
+
+declare -A EXIT_CODES
+ALL_PASSED=true
+
+run_service_test() {
+  local name=$1
+  local script=$2
+  local env_vars=$3
+
+  echo ""
+  echo "=== Running $name Load Test ==="
+
+  set +e
+  eval "$env_vars" k6 run $K6_OPTS "$script"
+  EXIT_CODE=$?
+  set -e
+
+  EXIT_CODES[$name]=$EXIT_CODE
+  if [[ $EXIT_CODE -ne 0 ]]; then
+    ALL_PASSED=false
+    echo "❌ $name load test FAILED (exit code: $EXIT_CODE)"
+  else
+    echo "✅ $name load test PASSED"
+  fi
+}
+
+if [[ "$SERVICE" == "all" || "$SERVICE" == "api" ]]; then
+  run_service_test "api" "${SCRIPT_DIR}/services/api.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS"
+fi
+
+if [[ "$SERVICE" == "all" || "$SERVICE" == "darkwatch" ]]; then
+  run_service_test "darkwatch" "${SCRIPT_DIR}/services/darkwatch.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS"
+fi
+
+if [[ "$SERVICE" == "all" || "$SERVICE" == "spamshield" ]]; then
+  run_service_test "spamshield" "${SCRIPT_DIR}/services/spamshield.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS"
+fi
+
+if [[ "$SERVICE" == "all" || "$SERVICE" == "voiceprint" ]]; then
+  run_service_test "voiceprint" "${SCRIPT_DIR}/services/voiceprint.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS"
+fi
+
+echo ""
+echo "=== Load Test Results ==="
+echo "Summary saved to: $SUMMARY_FILE"
+
+# Aggregate thresholds
+jq -n \
+  --arg timestamp "$TIMESTAMP" \
+  --arg base_url "$BASE_URL" \
+  --arg target_rps "$TARGET_RPS" \
+  --argjson exit_codes "$(printf '%s\n' "${!EXIT_CODES[@]}" | jq -R . | jq -s 'with_entries(.key = .[0])' | while read key; do echo; done)" \
+  '{timestamp: $timestamp, base_url: $base_url, target_rps: $target_rps, services: $exit_codes}' \
+  > "$THRESHOLD_FILE" 2>/dev/null || true
+
+for service in "${!EXIT_CODES[@]}"; do
+  status="pass"
+  [[ ${EXIT_CODES[$service]} -ne 0 ]] && status="fail"
+  echo "$service: $status"
+done
+
+echo ""
+if $ALL_PASSED; then
+  echo "✅ All load tests passed"
+  exit 0
+else
+  echo "❌ Some load tests failed"
+  exit 1
+fi

scripts/load-test/services/api.js

@@ -0,0 +1,48 @@
+import http from 'k6/http';
+import { check, group } from 'k6';
+import { Rate, Trend } from 'k6/metrics';
+import { getBaseUrl, getTargetRps, getDuration, defaultThresholds, checkResponse, randomString } from '../lib/common.js';
+
+const errorRate = new Rate('errors');
+const notificationLatency = new Trend('notification_p99');
+const correlationLatency = new Trend('correlation_p99');
+
+export const options = {
+  thresholds: {
+    ...defaultThresholds(250).thresholds,
+    notification_p99: ['p(99)<500'],
+    correlation_p99: ['p(99)<300'],
+  },
+};
+
+const BASE_URL = getBaseUrl();
+const AUTH_TOKEN = __ENV.API_TOKEN || 'test-token';
+
+const headers = {
+  'Content-Type': 'application/json',
+  'Authorization': `Bearer ${AUTH_TOKEN}`,
+};
+
+export default function () {
+  group('API Health', function () {
+    const res = http.get(`${BASE_URL}/health`, { headers });
+    checkResponse(res, 200);
+  });
+
+  group('Notifications', function () {
+    const payload = JSON.stringify({
+      userId: `user-${randomString()}`,
+      channel: 'email',
+      message: 'Load test notification',
+    });
+    const res = http.post(`${BASE_URL}/notifications`, payload, { headers });
+    checkResponse(res, 200);
+    notificationLatency.add(res.timings.duration);
+  });
+
+  group('Correlation', function () {
+    const res = http.get(`${BASE_URL}/correlation/events?limit=10`, { headers });
+    checkResponse(res, 200);
+    correlationLatency.add(res.timings.duration);
+  });
+}

scripts/load-test/services/darkwatch.js

@@ -0,0 +1,55 @@
+import http from 'k6/http';
+import { check, group } from 'k6';
+import { Rate, Trend } from 'k6/metrics';
+import { getBaseUrl, getTargetRps, getDuration, defaultThresholds, checkResponse, randomString } from '../lib/common.js';
+
+const errorRate = new Rate('errors');
+const scanLatency = new Trend('scan_p99');
+const watchlistLatency = new Trend('watchlist_p99');
+const alertLatency = new Trend('alert_p99');
+
+export const options = {
+  thresholds: {
+    ...defaultThresholds(200).thresholds,
+    scan_p99: ['p(99)<300'],
+    watchlist_p99: ['p(99)<200'],
+    alert_p99: ['p(99)<250'],
+  },
+};
+
+const BASE_URL = getBaseUrl();
+const AUTH_TOKEN = __ENV.API_TOKEN || 'test-token';
+
+const headers = {
+  'Content-Type': 'application/json',
+  'Authorization': `Bearer ${AUTH_TOKEN}`,
+};
+
+export default function () {
+  group('Darkwatch Scan', function () {
+    const payload = JSON.stringify({
+      type: 'email',
+      value: `loadtest-${randomString()}@example.com`,
+    });
+    const res = http.post(`${BASE_URL}/darkwatch/scan`, payload, { headers });
+    checkResponse(res, 200);
+    scanLatency.add(res.timings.duration);
+  });
+
+  group('Watchlist', function () {
+    const res = http.get(`${BASE_URL}/watchlist?page=1&limit=20`, { headers });
+    checkResponse(res, 200);
+    watchlistLatency.add(res.timings.duration);
+  });
+
+  group('Alerts', function () {
+    const res = http.get(`${BASE_URL}/alerts?status=open&limit=10`, { headers });
+    checkResponse(res, 200);
+    alertLatency.add(res.timings.duration);
+  });
+
+  group('Exposure Check', function () {
+    const res = http.get(`${BASE_URL}/exposure/summary`, { headers });
+    checkResponse(res, 200);
+  });
+}

scripts/load-test/services/spamshield.js

@@ -0,0 +1,48 @@
+import http from 'k6/http';
+import { check, group } from 'k6';
+import { Rate, Trend } from 'k6/metrics';
+import { getBaseUrl, getTargetRps, getDuration, defaultThresholds, checkResponse, randomString } from '../lib/common.js';
+
+const errorRate = new Rate('errors');
+const classificationLatency = new Trend('classification_p99');
+const engineLatency = new Trend('engine_p99');
+
+export const options = {
+  thresholds: {
+    ...defaultThresholds(200).thresholds,
+    classification_p99: ['p(99)<300'],
+    engine_p99: ['p(99)<250'],
+  },
+};
+
+const BASE_URL = getBaseUrl();
+const AUTH_TOKEN = __ENV.API_TOKEN || 'test-token';
+
+const headers = {
+  'Content-Type': 'application/json',
+  'Authorization': `Bearer ${AUTH_TOKEN}`,
+};
+
+export default function () {
+  group('SMS Classification', function () {
+    const payload = JSON.stringify({
+      message: `Load test message ${randomString(8)} - please check if this is spam`,
+      from: `+1555${String(Math.floor(1000000 + Math.random() * 9000000))}`,
+      to: `+1555${String(Math.floor(1000000 + Math.random() * 9000000))}`,
+    });
+    const res = http.post(`${BASE_URL}/spamshield/classify`, payload, { headers });
+    checkResponse(res, 200);
+    classificationLatency.add(res.timings.duration);
+  });
+
+  group('Spam Engine Health', function () {
+    const res = http.get(`${BASE_URL}/spamshield/health`, { headers });
+    checkResponse(res, 200);
+    engineLatency.add(res.timings.duration);
+  });
+
+  group('Blocklist Check', function () {
+    const res = http.get(`${BASE_URL}/spamshield/blocklist/check?phone=+1555${String(Math.floor(1000000 + Math.random() * 9000000))}`, { headers });
+    checkResponse(res, 200);
+  });
+}

scripts/load-test/services/voiceprint.js

@@ -0,0 +1,56 @@
+import http from 'k6/http';
+import { check, group } from 'k6';
+import { Rate, Trend } from 'k6/metrics';
+import { getBaseUrl, getTargetRps, getDuration, defaultThresholds, checkResponse, randomString } from '../lib/common.js';
+
+const errorRate = new Rate('errors');
+const enrollmentLatency = new Trend('enrollment_p99');
+const verificationLatency = new Trend('verification_p99');
+const modelLatency = new Trend('model_retrieval_p99');
+
+export const options = {
+  thresholds: {
+    ...defaultThresholds(250).thresholds,
+    enrollment_p99: ['p(99)<500'],
+    verification_p99: ['p(99)<250'],
+    model_retrieval_p99: ['p(99)<100'],
+  },
+};
+
+const BASE_URL = getBaseUrl();
+const AUTH_TOKEN = __ENV.API_TOKEN || 'test-token';
+
+const headers = {
+  'Content-Type': 'application/json',
+  'Authorization': `Bearer ${AUTH_TOKEN}`,
+};
+
+export default function () {
+  group('Voiceprint Enrollment', function () {
+    const payload = JSON.stringify({
+      userId: `loadtest-${randomString()}`,
+      audioSample: 'base64-encoded-audio-data-placeholder',
+      sampleRate: 16000,
+    });
+    const res = http.post(`${BASE_URL}/voiceprint/enroll`, payload, { headers });
+    checkResponse(res, 200);
+    enrollmentLatency.add(res.timings.duration);
+  });
+
+  group('Voiceprint Verification', function () {
+    const payload = JSON.stringify({
+      userId: `loadtest-${randomString()}`,
+      audioSample: 'base64-encoded-audio-data-placeholder',
+      sampleRate: 16000,
+    });
+    const res = http.post(`${BASE_URL}/voiceprint/verify`, payload, { headers });
+    checkResponse(res, 200);
+    verificationLatency.add(res.timings.duration);
+  });
+
+  group('Model Retrieval', function () {
+    const res = http.get(`${BASE_URL}/voiceprint/model/${randomString()}`, { headers });
+    checkResponse(res, 200);
+    modelLatency.add(res.timings.duration);
+  });
+}