diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f0434b..b5c994d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,14 +217,13 @@ jobs: if: always() run: | if [ -f scripts/load-test/reports/threshold-results.json ]; then - FAILURES=$(jq -r '.metrics | to_entries[] | select(.value.passes == false) | .key' scripts/load-test/reports/threshold-results.json) - if [ -n "$FAILURES" ]; then - echo "❌ P99 threshold failures:" - echo "$FAILURES" + FAILURES=$(jq -r '[.services | to_entries[] | select(.value.exitCode != 0) | .key] | join(", ")' scripts/load-test/reports/threshold-results.json 2>/dev/null || echo "") + if [ -n "$FAILURES" ] && [ "$FAILURES" != "" ]; then + echo "❌ Load test failures: $FAILURES" exit 1 else - echo "✅ All P99 thresholds passed" + echo "✅ All load tests passed" fi else - echo "No threshold results file found" + echo "⚠️ No threshold results file found" fi diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml index 332e102..4816af1 100644 --- a/.github/workflows/load-test.yml +++ b/.github/workflows/load-test.yml @@ -64,14 +64,13 @@ jobs: if: always() run: | if [ -f scripts/load-test/reports/threshold-results.json ]; then - FAILURES=$(jq -r '.metrics | to_entries[] | select(.value.passes == false) | .key' scripts/load-test/reports/threshold-results.json 2>/dev/null || echo "") - if [ -n "$FAILURES" ]; then - echo "❌ P99 threshold failures:" - echo "$FAILURES" + FAILURES=$(jq -r '[.services | to_entries[] | select(.value.exitCode != 0) | .key] | join(", ")' scripts/load-test/reports/threshold-results.json 2>/dev/null || echo "") + if [ -n "$FAILURES" ] && [ "$FAILURES" != "" ]; then + echo "❌ Load test failures: $FAILURES" exit 1 else - echo "✅ All P99 thresholds passed" + echo "✅ All load tests passed" fi else - echo "No threshold results file found" + echo "⚠️ No threshold results file found" fi diff --git a/scripts/load-test/lib/common.js b/scripts/load-test/lib/common.js index efe09cf..e3291fb 100644 --- a/scripts/load-test/lib/common.js +++ b/scripts/load-test/lib/common.js @@ -16,8 +16,10 @@ export function getDuration() { export function defaultThresholds(p99ms) { return { - http_req_duration: [`p(99)<${p99ms}`], - errors: ['rate<0.01'], + thresholds: { + http_req_duration: [`p(99)<${p99ms}`], + errors: ['rate<0.01'], + }, }; } diff --git a/scripts/load-test/run-all.sh b/scripts/load-test/run-all.sh index ddba5de..b573048 100755 --- a/scripts/load-test/run-all.sh +++ b/scripts/load-test/run-all.sh @@ -8,7 +8,6 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPORT_DIR="${SCRIPT_DIR}/reports" TIMESTAMP="$(date +%Y%m%d-%H%M%S)" -SUMMARY_FILE="${REPORT_DIR}/summary-${TIMESTAMP}.json" THRESHOLD_FILE="${REPORT_DIR}/threshold-results.json" SERVICE="${1:-all}" @@ -17,6 +16,7 @@ mkdir -p "$REPORT_DIR" BASE_URL="${LOAD_TEST_BASE_URL:-http://localhost:3000}" TARGET_RPS="${TARGET_RPS:-500}" DURATION="${DURATION:-300s}" +API_TOKEN="${API_TOKEN:-test-token}" echo "=== ShieldAI Combined Load Test ===" echo "Timestamp: $TIMESTAMP" @@ -26,7 +26,7 @@ echo "Duration: $DURATION" echo "Service: $SERVICE" echo "" -K6_OPTS="--duration $DURATION --summary-export $SUMMARY_FILE" +K6_OPTS="--summary-export ${REPORT_DIR}/summary-${TIMESTAMP}.json" if [[ -n "${K6_CLOUD_TOKEN:-}" ]]; then K6_OPTS="$K6_OPTS --out cloud" @@ -35,17 +35,23 @@ fi declare -A EXIT_CODES ALL_PASSED=true +SERVICE_ENV="BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS DURATION=$DURATION API_TOKEN=$API_TOKEN" run_service_test() { local name=$1 local script=$2 - local env_vars=$3 + local summary_file="${REPORT_DIR}/${name}-summary-${TIMESTAMP}.json" echo "" echo "=== Running $name Load Test ===" + local opts="--summary-export $summary_file" + if [[ -n "${K6_CLOUD_TOKEN:-}" ]]; then + opts="$opts --out cloud" + fi + set +e - eval "$env_vars" k6 run $K6_OPTS "$script" + eval "$SERVICE_ENV" k6 run $opts "$script" EXIT_CODE=$? set -e @@ -59,33 +65,49 @@ run_service_test() { } if [[ "$SERVICE" == "all" || "$SERVICE" == "api" ]]; then - run_service_test "api" "${SCRIPT_DIR}/services/api.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS" + run_service_test "api" "${SCRIPT_DIR}/services/api.js" fi if [[ "$SERVICE" == "all" || "$SERVICE" == "darkwatch" ]]; then - run_service_test "darkwatch" "${SCRIPT_DIR}/services/darkwatch.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS" + run_service_test "darkwatch" "${SCRIPT_DIR}/services/darkwatch.js" fi if [[ "$SERVICE" == "all" || "$SERVICE" == "spamshield" ]]; then - run_service_test "spamshield" "${SCRIPT_DIR}/services/spamshield.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS" + run_service_test "spamshield" "${SCRIPT_DIR}/services/spamshield.js" fi if [[ "$SERVICE" == "all" || "$SERVICE" == "voiceprint" ]]; then - run_service_test "voiceprint" "${SCRIPT_DIR}/services/voiceprint.js" "BASE_URL=$BASE_URL TARGET_RPS=$TARGET_RPS" + run_service_test "voiceprint" "${SCRIPT_DIR}/services/voiceprint.js" fi +# Aggregate threshold results from all service summaries echo "" echo "=== Load Test Results ===" -echo "Summary saved to: $SUMMARY_FILE" -# Aggregate thresholds -jq -n \ - --arg timestamp "$TIMESTAMP" \ - --arg base_url "$BASE_URL" \ - --arg target_rps "$TARGET_RPS" \ - --argjson exit_codes "$(printf '%s\n' "${!EXIT_CODES[@]}" | jq -R . | jq -s 'with_entries(.key = .[0])' | while read key; do echo; done)" \ - '{timestamp: $timestamp, base_url: $base_url, target_rps: $target_rps, services: $exit_codes}' \ - > "$THRESHOLD_FILE" 2>/dev/null || true +# Build threshold-results.json from k6 summary exports +jq -n --arg timestamp "$TIMESTAMP" --arg base_url "$BASE_URL" --arg target_rps "$TARGET_RPS" \ + '{timestamp: $timestamp, base_url: $base_url, target_rps: $target_rps, services: {}}' \ + > "$THRESHOLD_FILE" + +for name in "${!EXIT_CODES[@]}"; do + summary_file="${REPORT_DIR}/${name}-summary-${TIMESTAMP}.json" + if [[ -f "$summary_file" ]]; then + jq --arg name "$name" --argjson exit_code "${EXIT_CODES[$name]}" \ + '.services[$name] = { + exitCode: $exit_code, + passed: ($exit_code == 0), + metrics: (input | .metrics // {}) + }' \ + "$THRESHOLD_FILE" "$summary_file" \ + > "${THRESHOLD_FILE}.tmp" && mv "${THRESHOLD_FILE}.tmp" "$THRESHOLD_FILE" + else + jq --arg name "$name" --argjson exit_code "${EXIT_CODES[$name]}" \ + '.services[$name] = {exitCode: $exit_code, passed: ($exit_code == 0)}' \ + "$THRESHOLD_FILE" > "${THRESHOLD_FILE}.tmp" && mv "${THRESHOLD_FILE}.tmp" "$THRESHOLD_FILE" + fi +done + +echo "Threshold results saved to: $THRESHOLD_FILE" for service in "${!EXIT_CODES[@]}"; do status="pass" diff --git a/scripts/load-test/services/api.js b/scripts/load-test/services/api.js index 14b6ef6..f3e9e0f 100644 --- a/scripts/load-test/services/api.js +++ b/scripts/load-test/services/api.js @@ -7,7 +7,22 @@ const errorRate = new Rate('errors'); const notificationLatency = new Trend('notification_p99'); const correlationLatency = new Trend('correlation_p99'); +const TARGET_RPS = getTargetRps(); +const DURATION = getDuration(); + export const options = { + scenarios: { + sustained_load: { + executor: 'constant-arrival-rate', + duration: DURATION, + rate: TARGET_RPS, + preAllocatedVUs: 20, + maxVUs: 100, + startTime: '0s', + exec: 'default', + tags: { scenario: 'sustained_load' }, + }, + }, thresholds: { ...defaultThresholds(250).thresholds, notification_p99: ['p(99)<500'], diff --git a/scripts/load-test/services/darkwatch.js b/scripts/load-test/services/darkwatch.js index 111c849..644aa9d 100644 --- a/scripts/load-test/services/darkwatch.js +++ b/scripts/load-test/services/darkwatch.js @@ -8,7 +8,22 @@ const scanLatency = new Trend('scan_p99'); const watchlistLatency = new Trend('watchlist_p99'); const alertLatency = new Trend('alert_p99'); +const TARGET_RPS = getTargetRps(); +const DURATION = getDuration(); + export const options = { + scenarios: { + sustained_load: { + executor: 'constant-arrival-rate', + duration: DURATION, + rate: TARGET_RPS, + preAllocatedVUs: 20, + maxVUs: 100, + startTime: '0s', + exec: 'default', + tags: { scenario: 'sustained_load' }, + }, + }, thresholds: { ...defaultThresholds(200).thresholds, scan_p99: ['p(99)<300'], diff --git a/scripts/load-test/services/spamshield.js b/scripts/load-test/services/spamshield.js index a4970a4..06c907a 100644 --- a/scripts/load-test/services/spamshield.js +++ b/scripts/load-test/services/spamshield.js @@ -8,7 +8,22 @@ const smsClassifyP99 = new Trend('sms_classify_p99'); const numberReputationP99 = new Trend('number_reputation_p99'); const callAnalyzeP99 = new Trend('call_analyze_p99'); +const TARGET_RPS = getTargetRps(); +const DURATION = getDuration(); + export const options = { + scenarios: { + sustained_load: { + executor: 'constant-arrival-rate', + duration: DURATION, + rate: TARGET_RPS, + preAllocatedVUs: 20, + maxVUs: 100, + startTime: '0s', + exec: 'default', + tags: { scenario: 'sustained_load' }, + }, + }, thresholds: { ...defaultThresholds(400).thresholds, sms_classify_p99: ['p(99)<150'], diff --git a/scripts/load-test/services/voiceprint.js b/scripts/load-test/services/voiceprint.js index 35e4b14..0347b44 100644 --- a/scripts/load-test/services/voiceprint.js +++ b/scripts/load-test/services/voiceprint.js @@ -8,7 +8,22 @@ const enrollmentLatency = new Trend('enrollment_p99'); const verificationLatency = new Trend('verification_p99'); const modelLatency = new Trend('model_retrieval_p99'); +const TARGET_RPS = getTargetRps(); +const DURATION = getDuration(); + export const options = { + scenarios: { + sustained_load: { + executor: 'constant-arrival-rate', + duration: DURATION, + rate: TARGET_RPS, + preAllocatedVUs: 20, + maxVUs: 100, + startTime: '0s', + exec: 'default', + tags: { scenario: 'sustained_load' }, + }, + }, thresholds: { ...defaultThresholds(250).thresholds, enrollment_p99: ['p(99)<500'],