From a804cab431f51804911af663da208cf34f6a1d05 Mon Sep 17 00:00:00 2001 From: Michael Freno Date: Sat, 9 May 2026 09:16:36 -0400 Subject: [PATCH] Add load testing job to GitHub Actions CI pipeline (FRE-4931) - Add load-test job to ci.yml that runs after docker-build on push to main - Create combined load test runner (scripts/load-test/run-all.sh) for all services - Create k6 load test scripts for api, darkwatch, spamshield, and voiceprint - Add shared k6 utilities (lib/common.js) - Update load-test.yml to support all services and report artifacts - Configure k6 cloud output and P99 threshold validation - Generate load test report as CI artifact Co-Authored-By: Paperclip --- .github/workflows/ci.yml | 51 +++++++++++ .github/workflows/load-test.yml | 49 ++++++++--- scripts/load-test/lib/common.js | 42 +++++++++ scripts/load-test/run-all.sh | 103 +++++++++++++++++++++++ scripts/load-test/services/api.js | 48 +++++++++++ scripts/load-test/services/darkwatch.js | 55 ++++++++++++ scripts/load-test/services/spamshield.js | 48 +++++++++++ scripts/load-test/services/voiceprint.js | 56 ++++++++++++ 8 files changed, 439 insertions(+), 13 deletions(-) create mode 100644 scripts/load-test/lib/common.js create mode 100755 scripts/load-test/run-all.sh create mode 100644 scripts/load-test/services/api.js create mode 100644 scripts/load-test/services/darkwatch.js create mode 100644 scripts/load-test/services/spamshield.js create mode 100644 scripts/load-test/services/voiceprint.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96c3cb3..0f0434b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,3 +177,54 @@ jobs: env: TF_VAR_hibp_api_key: ${{ secrets.HIBP_API_KEY }} TF_VAR_resend_api_key: ${{ secrets.RESEND_API_KEY }} + + load-test: + name: Load Test + runs-on: ubuntu-latest + needs: [lint, typecheck, test, docker-build] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + environment: staging + steps: + - uses: actions/checkout@v4 + + - name: Install k6 + run: | + curl -s https://github.com/grafana/k6/releases/download/v0.50.0/k6-linux-amd64.tar.gz -L | tar xz + sudo mv k6 /usr/local/bin/ + k6 version + + - name: Run combined load tests + run: | + chmod +x scripts/load-test/run-all.sh + ./scripts/load-test/run-all.sh + env: + LOAD_TEST_BASE_URL: ${{ secrets.LOAD_TEST_BASE_URL || 'http://localhost:3000' }} + API_TOKEN: ${{ secrets.LOAD_TEST_API_TOKEN || 'test-token' }} + TARGET_RPS: ${{ vars.LOAD_TEST_TARGET_RPS || '500' }} + DURATION: ${{ vars.LOAD_TEST_DURATION || '300s' }} + K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN || '' }} + K6_CLOUD_PROJECT_ID: ${{ vars.K6_CLOUD_PROJECT_ID || '' }} + + - name: Upload load test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: load-test-report-${{ github.sha }} + path: scripts/load-test/reports/ + retention-days: 30 + + - name: Check P99 thresholds + if: always() + run: | + if [ -f scripts/load-test/reports/threshold-results.json ]; then + FAILURES=$(jq -r '.metrics | to_entries[] | select(.value.passes == false) | .key' scripts/load-test/reports/threshold-results.json) + if [ -n "$FAILURES" ]; then + echo "❌ P99 threshold failures:" + echo "$FAILURES" + exit 1 + else + echo "✅ All P99 thresholds passed" + fi + else + echo "No threshold results file found" + fi diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml index 2d56f5e..332e102 100644 --- a/.github/workflows/load-test.yml +++ b/.github/workflows/load-test.yml @@ -13,16 +13,24 @@ on: description: 'Test duration' required: false default: '300s' + service: + description: 'Service to test (all, api, darkwatch, spamshield, voiceprint)' + required: false + default: 'all' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + NODE_VERSION: "20" + jobs: - load-test-voiceprint: - name: Voiceprint Load Test + load-test: + name: Load Test (${{ github.event.inputs.service || 'all' }}) runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 + environment: staging steps: - uses: actions/checkout@v4 @@ -32,23 +40,38 @@ jobs: sudo mv k6 /usr/local/bin/ k6 version - - name: Run Voiceprint load tests + - name: Run load tests run: | - cd load-tests/voiceprint - ./run.sh mixed + chmod +x scripts/load-test/run-all.sh + ./scripts/load-test/run-all.sh ${{ github.event.inputs.service || 'all' }} env: - VOICEPRINT_BASE_URL: ${{ secrets.LOAD_TEST_BASE_URL || 'http://localhost:3000' }} + LOAD_TEST_BASE_URL: ${{ secrets.LOAD_TEST_BASE_URL || 'http://localhost:3000' }} API_TOKEN: ${{ secrets.LOAD_TEST_API_TOKEN || 'test-token' }} TARGET_RPS: ${{ github.event.inputs.target_rps || '500' }} DURATION: ${{ github.event.inputs.duration || '300s' }} - ENROLLMENT_P99_MS: '500' - VERIFICATION_P99_MS: '250' - MODEL_RETRIEVAL_P99_MS: '100' + K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN || '' }} + K6_CLOUD_PROJECT_ID: ${{ vars.K6_CLOUD_PROJECT_ID || '' }} - - name: Upload results + - name: Upload load test report if: always() uses: actions/upload-artifact@v4 with: - name: load-test-results-${{ github.sha }} - path: load-tests/voiceprint/results/ + name: load-test-report-${{ github.sha }} + path: scripts/load-test/reports/ retention-days: 30 + + - name: Check P99 thresholds + if: always() + run: | + if [ -f scripts/load-test/reports/threshold-results.json ]; then + FAILURES=$(jq -r '.metrics | to_entries[] | select(.value.passes == false) | .key' scripts/load-test/reports/threshold-results.json 2>/dev/null || echo "") + if [ -n "$FAILURES" ]; then + echo "❌ P99 threshold failures:" + echo "$FAILURES" + exit 1 + else + echo "✅ All P99 thresholds passed" + fi + else + echo "No threshold results file found" + fi diff --git a/scripts/load-test/lib/common.js b/scripts/load-test/lib/common.js new file mode 100644 index 0000000..efe09cf --- /dev/null +++ b/scripts/load-test/lib/common.js @@ -0,0 +1,42 @@ +import { Rate, Trend } from 'k6/metrics'; + +export const errorRate = new Rate('errors'); + +export function getBaseUrl() { + return __ENV.BASE_URL || 'http://localhost:3000'; +} + +export function getTargetRps() { + return parseInt(__ENV.TARGET_RPS || '500', 10); +} + +export function getDuration() { + return __ENV.DURATION || '300s'; +} + +export function defaultThresholds(p99ms) { + return { + http_req_duration: [`p(99)<${p99ms}`], + errors: ['rate<0.01'], + }; +} + +export function checkResponse(res, expectedStatus = 200) { + const pass = check(res, { + 'status is expected': (r) => r.status === expectedStatus, + 'response time OK': (r) => r.timings.duration < 5000, + }); + if (!pass) { + errorRate.add(1); + } + return pass; +} + +export function randomString(length = 10) { + const chars = 'abcdefghijklmnopqrstuvwxyz0123456789'; + let result = ''; + for (let i = 0; i < length; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return result; +} diff --git a/scripts/load-test/run-all.sh b/scripts/load-test/run-all.sh new file mode 100755 index 0000000..ddba5de --- /dev/null +++ b/scripts/load-test/run-all.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Combined load test runner for all ShieldAI services +# Usage: ./run-all.sh [service] +# service: all (default), api, darkwatch, spamshield, voiceprint + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPORT_DIR="${SCRIPT_DIR}/reports" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +SUMMARY_FILE="${REPORT_DIR}/summary-${TIMESTAMP}.json" +THRESHOLD_FILE="${REPORT_DIR}/threshold-results.json" +SERVICE="${1:-all}" + +mkdir -p "$REPORT_DIR" + +BASE_URL="${LOAD_TEST_BASE_URL:-http://localhost:3000}" +TARGET_RPS="${TARGET_RPS:-500}" +DURATION="${DURATION:-300s}" + +echo "=== ShieldAI Combined Load Test ===" +echo "Timestamp: $TIMESTAMP" +echo "Base URL: $BASE_URL" +echo "Target RPS: $TARGET_RPS" +echo "Duration: $DURATION" +echo "Service: $SERVICE" +echo "" + +K6_OPTS="--duration $DURATION --summary-export $SUMMARY_FILE" + +if [[ -n "${K6_CLOUD_TOKEN:-}" ]]; then + K6_OPTS="$K6_OPTS --out cloud" + echo "k6 cloud output: enabled" +fi + +declare -A EXIT_CODES +ALL_PASSED=true + +run_service_test() { + local name=$1 + local script=$2 + local env_vars=$3 + + echo "" + echo "=== Running $name Load Test ===" + + set +e + eval "$env_vars" k6 run $K6_OPTS "$script" + EXIT_CODE=$? + set -e + + EXIT_CODES[$name]=$EXIT_CODE + if [[ $EXIT_CODE -ne 0 ]]; then + ALL_PASSED=false + echo "❌ $name load test FAILED (exit code: $EXIT_CODE)" + else + echo "✅ $name load test PASSED" + fi +} + +if [[ "$SERVICE" == "all" || "$SERVICE" == "api" ]]; then + run_service_test "api" "${SCRIPT_DIR}/services/api.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS" +fi + +if [[ "$SERVICE" == "all" || "$SERVICE" == "darkwatch" ]]; then + run_service_test "darkwatch" "${SCRIPT_DIR}/services/darkwatch.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS" +fi + +if [[ "$SERVICE" == "all" || "$SERVICE" == "spamshield" ]]; then + run_service_test "spamshield" "${SCRIPT_DIR}/services/spamshield.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS" +fi + +if [[ "$SERVICE" == "all" || "$SERVICE" == "voiceprint" ]]; then + run_service_test "voiceprint" "${SCRIPT_DIR}/services/voiceprint.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS" +fi + +echo "" +echo "=== Load Test Results ===" +echo "Summary saved to: $SUMMARY_FILE" + +# Aggregate thresholds +jq -n \ + --arg timestamp "$TIMESTAMP" \ + --arg base_url "$BASE_URL" \ + --arg target_rps "$TARGET_RPS" \ + --argjson exit_codes "$(printf '%s\n' "${!EXIT_CODES[@]}" | jq -R . | jq -s 'with_entries(.key = .[0])' | while read key; do echo; done)" \ + '{timestamp: $timestamp, base_url: $base_url, target_rps: $target_rps, services: $exit_codes}' \ + > "$THRESHOLD_FILE" 2>/dev/null || true + +for service in "${!EXIT_CODES[@]}"; do + status="pass" + [[ ${EXIT_CODES[$service]} -ne 0 ]] && status="fail" + echo "$service: $status" +done + +echo "" +if $ALL_PASSED; then + echo "✅ All load tests passed" + exit 0 +else + echo "❌ Some load tests failed" + exit 1 +fi diff --git a/scripts/load-test/services/api.js b/scripts/load-test/services/api.js new file mode 100644 index 0000000..14b6ef6 --- /dev/null +++ b/scripts/load-test/services/api.js @@ -0,0 +1,48 @@ +import http from 'k6/http'; +import { check, group } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; +import { getBaseUrl, getTargetRps, getDuration, defaultThresholds, checkResponse, randomString } from '../lib/common.js'; + +const errorRate = new Rate('errors'); +const notificationLatency = new Trend('notification_p99'); +const correlationLatency = new Trend('correlation_p99'); + +export const options = { + thresholds: { + ...defaultThresholds(250).thresholds, + notification_p99: ['p(99)<500'], + correlation_p99: ['p(99)<300'], + }, +}; + +const BASE_URL = getBaseUrl(); +const AUTH_TOKEN = __ENV.API_TOKEN || 'test-token'; + +const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${AUTH_TOKEN}`, +}; + +export default function () { + group('API Health', function () { + const res = http.get(`${BASE_URL}/health`, { headers }); + checkResponse(res, 200); + }); + + group('Notifications', function () { + const payload = JSON.stringify({ + userId: `user-${randomString()}`, + channel: 'email', + message: 'Load test notification', + }); + const res = http.post(`${BASE_URL}/notifications`, payload, { headers }); + checkResponse(res, 200); + notificationLatency.add(res.timings.duration); + }); + + group('Correlation', function () { + const res = http.get(`${BASE_URL}/correlation/events?limit=10`, { headers }); + checkResponse(res, 200); + correlationLatency.add(res.timings.duration); + }); +} diff --git a/scripts/load-test/services/darkwatch.js b/scripts/load-test/services/darkwatch.js new file mode 100644 index 0000000..111c849 --- /dev/null +++ b/scripts/load-test/services/darkwatch.js @@ -0,0 +1,55 @@ +import http from 'k6/http'; +import { check, group } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; +import { getBaseUrl, getTargetRps, getDuration, defaultThresholds, checkResponse, randomString } from '../lib/common.js'; + +const errorRate = new Rate('errors'); +const scanLatency = new Trend('scan_p99'); +const watchlistLatency = new Trend('watchlist_p99'); +const alertLatency = new Trend('alert_p99'); + +export const options = { + thresholds: { + ...defaultThresholds(200).thresholds, + scan_p99: ['p(99)<300'], + watchlist_p99: ['p(99)<200'], + alert_p99: ['p(99)<250'], + }, +}; + +const BASE_URL = getBaseUrl(); +const AUTH_TOKEN = __ENV.API_TOKEN || 'test-token'; + +const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${AUTH_TOKEN}`, +}; + +export default function () { + group('Darkwatch Scan', function () { + const payload = JSON.stringify({ + type: 'email', + value: `loadtest-${randomString()}@example.com`, + }); + const res = http.post(`${BASE_URL}/darkwatch/scan`, payload, { headers }); + checkResponse(res, 200); + scanLatency.add(res.timings.duration); + }); + + group('Watchlist', function () { + const res = http.get(`${BASE_URL}/watchlist?page=1&limit=20`, { headers }); + checkResponse(res, 200); + watchlistLatency.add(res.timings.duration); + }); + + group('Alerts', function () { + const res = http.get(`${BASE_URL}/alerts?status=open&limit=10`, { headers }); + checkResponse(res, 200); + alertLatency.add(res.timings.duration); + }); + + group('Exposure Check', function () { + const res = http.get(`${BASE_URL}/exposure/summary`, { headers }); + checkResponse(res, 200); + }); +} diff --git a/scripts/load-test/services/spamshield.js b/scripts/load-test/services/spamshield.js new file mode 100644 index 0000000..a63ec92 --- /dev/null +++ b/scripts/load-test/services/spamshield.js @@ -0,0 +1,48 @@ +import http from 'k6/http'; +import { check, group } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; +import { getBaseUrl, getTargetRps, getDuration, defaultThresholds, checkResponse, randomString } from '../lib/common.js'; + +const errorRate = new Rate('errors'); +const classificationLatency = new Trend('classification_p99'); +const engineLatency = new Trend('engine_p99'); + +export const options = { + thresholds: { + ...defaultThresholds(200).thresholds, + classification_p99: ['p(99)<300'], + engine_p99: ['p(99)<250'], + }, +}; + +const BASE_URL = getBaseUrl(); +const AUTH_TOKEN = __ENV.API_TOKEN || 'test-token'; + +const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${AUTH_TOKEN}`, +}; + +export default function () { + group('SMS Classification', function () { + const payload = JSON.stringify({ + message: `Load test message ${randomString(8)} - please check if this is spam`, + from: `+1555${String(Math.floor(1000000 + Math.random() * 9000000))}`, + to: `+1555${String(Math.floor(1000000 + Math.random() * 9000000))}`, + }); + const res = http.post(`${BASE_URL}/spamshield/classify`, payload, { headers }); + checkResponse(res, 200); + classificationLatency.add(res.timings.duration); + }); + + group('Spam Engine Health', function () { + const res = http.get(`${BASE_URL}/spamshield/health`, { headers }); + checkResponse(res, 200); + engineLatency.add(res.timings.duration); + }); + + group('Blocklist Check', function () { + const res = http.get(`${BASE_URL}/spamshield/blocklist/check?phone=+1555${String(Math.floor(1000000 + Math.random() * 9000000))}`, { headers }); + checkResponse(res, 200); + }); +} diff --git a/scripts/load-test/services/voiceprint.js b/scripts/load-test/services/voiceprint.js new file mode 100644 index 0000000..35e4b14 --- /dev/null +++ b/scripts/load-test/services/voiceprint.js @@ -0,0 +1,56 @@ +import http from 'k6/http'; +import { check, group } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; +import { getBaseUrl, getTargetRps, getDuration, defaultThresholds, checkResponse, randomString } from '../lib/common.js'; + +const errorRate = new Rate('errors'); +const enrollmentLatency = new Trend('enrollment_p99'); +const verificationLatency = new Trend('verification_p99'); +const modelLatency = new Trend('model_retrieval_p99'); + +export const options = { + thresholds: { + ...defaultThresholds(250).thresholds, + enrollment_p99: ['p(99)<500'], + verification_p99: ['p(99)<250'], + model_retrieval_p99: ['p(99)<100'], + }, +}; + +const BASE_URL = getBaseUrl(); +const AUTH_TOKEN = __ENV.API_TOKEN || 'test-token'; + +const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${AUTH_TOKEN}`, +}; + +export default function () { + group('Voiceprint Enrollment', function () { + const payload = JSON.stringify({ + userId: `loadtest-${randomString()}`, + audioSample: 'base64-encoded-audio-data-placeholder', + sampleRate: 16000, + }); + const res = http.post(`${BASE_URL}/voiceprint/enroll`, payload, { headers }); + checkResponse(res, 200); + enrollmentLatency.add(res.timings.duration); + }); + + group('Voiceprint Verification', function () { + const payload = JSON.stringify({ + userId: `loadtest-${randomString()}`, + audioSample: 'base64-encoded-audio-data-placeholder', + sampleRate: 16000, + }); + const res = http.post(`${BASE_URL}/voiceprint/verify`, payload, { headers }); + checkResponse(res, 200); + verificationLatency.add(res.timings.duration); + }); + + group('Model Retrieval', function () { + const res = http.get(`${BASE_URL}/voiceprint/model/${randomString()}`, { headers }); + checkResponse(res, 200); + modelLatency.add(res.timings.duration); + }); +}