#!/bin/bash set -uo pipefail # ShieldAI Rollback Test Suite # Usage: ./test-rollback.sh [ecs|compose|migration|all] # # Validates rollback scripts and procedures without mutating production # Run against staging environment for integration tests TEST_SUITE="${1:-all}" PASS=0 FAIL=0 SKIP=0 # ─── Helpers ───────────────────────────────────────────────────── log() { echo "[$(date -u '+%H:%M:%S')] $*" } assert_eq() { local desc="$1" expected="$2" actual="$3" if [[ "$expected" == "$actual" ]]; then log " ✅ PASS: $desc" ((PASS++)) else log " ❌ FAIL: $desc (expected: $expected, got: $actual)" ((FAIL++)) fi } assert_file_exists() { local desc="$1" path="$2" if [[ -f "$path" ]]; then log " ✅ PASS: $desc" ((PASS++)) else log " ❌ FAIL: $desc ($path not found)" ((FAIL++)) fi } assert_executable() { local desc="$1" path="$2" if [[ -x "$path" ]]; then log " ✅ PASS: $desc" ((PASS++)) else log " ❌ FAIL: $desc ($path not executable)" ((FAIL++)) fi } assert_script_syntax() { local desc="$1" path="$2" if bash -n "$path" 2>/dev/null; then log " ✅ PASS: $desc (syntax OK)" ((PASS++)) else log " ❌ FAIL: $desc (syntax error)" ((FAIL++)) fi } assert_contains() { local desc="$1" file="$2" pattern="$3" if grep -q -- "$pattern" "$file" 2>/dev/null; then log " ✅ PASS: $desc" ((PASS++)) else log " ❌ FAIL: $desc (pattern '$pattern' not found in $file)" ((FAIL++)) fi } # ─── Test: File Structure ──────────────────────────────────────── test_file_structure() { log "=== Test: File Structure ===" assert_file_exists "ROLLBACK.md exists" "infra/ROLLBACK.md" assert_file_exists "rollback.sh exists" "infra/scripts/rollback.sh" assert_file_exists "rollback-compose.sh exists" "infra/scripts/rollback-compose.sh" assert_file_exists "rollback-migration.sh exists" "infra/scripts/rollback-migration.sh" assert_executable "rollback.sh is executable" "infra/scripts/rollback.sh" assert_executable "rollback-compose.sh is executable" "infra/scripts/rollback-compose.sh" assert_executable "rollback-migration.sh is executable" "infra/scripts/rollback-migration.sh" } # ─── Test: Script Syntax ───────────────────────────────────────── test_script_syntax() { log "=== Test: Script Syntax ===" assert_script_syntax "rollback.sh syntax" "infra/scripts/rollback.sh" assert_script_syntax "rollback-compose.sh syntax" "infra/scripts/rollback-compose.sh" assert_script_syntax "rollback-migration.sh syntax" "infra/scripts/rollback-migration.sh" } # ─── Test: ROLLBACK.md Content ─────────────────────────────────── test_documentation() { log "=== Test: Documentation Content ===" local doc="infra/ROLLBACK.md" for section in "Overview" "ECS Service Rollback" "Docker Compose Rollback" \ "Database Migration Rollback" "Automated Rollback Triggers" \ "Blue-Green Deployment Rollback" "Rollback Decision Tree" \ "Post-Rollback Verification" "Testing Checklist" "Emergency Rollback"; do assert_contains "Section '$section' documented" "$doc" "$section" done for cmd in "aws ecs update-service" "docker compose" "drizzle-kit" \ "aws rds restore-db-instance" "aws ecs wait services-stable"; do assert_contains "Command '$cmd' documented" "$doc" "$cmd" done } # ─── Test: Rollback Script Validation ──────────────────────────── test_rollback_script() { log "=== Test: ECS Rollback Script ===" # Test invalid environment local exit_code=0 bash infra/scripts/rollback.sh invalid_env api >/dev/null 2>&1 || exit_code=$? assert_eq "Invalid environment returns exit code 1" "1" "$exit_code" # Test invalid service exit_code=0 bash infra/scripts/rollback.sh staging invalid_svc >/dev/null 2>&1 || exit_code=$? assert_eq "Invalid service returns exit code 1" "1" "$exit_code" # Verify script has required functions for func in "validate_environment" "validate_service" "rollback_service" \ "verify_health" "check_prerequisites" "main"; do assert_contains "Function '$func' defined" "infra/scripts/rollback.sh" "$func" done # Verify all services are handled for svc in api darkwatch spamshield voiceprint; do assert_contains "Service '$svc' in SERVICES_LIST" "infra/scripts/rollback.sh" "$svc" done } # ─── Test: Compose Rollback Script ─────────────────────────────── test_compose_script() { log "=== Test: Docker Compose Rollback Script ===" # Test missing tag argument local exit_code=0 bash infra/scripts/rollback-compose.sh >/dev/null 2>&1 || exit_code=$? assert_eq "Missing tag returns exit code 1" "1" "$exit_code" # Verify compose file exists assert_file_exists "docker-compose.prod.yml exists" "docker-compose.prod.yml" # Verify all services are defined in compose for svc in api darkwatch spamshield voiceprint; do assert_contains "Service '$svc' in docker-compose.prod.yml" "docker-compose.prod.yml" " ${svc}:" done } # ─── Test: CI/CD Rollback Job ──────────────────────────────────── test_cicd_rollback() { log "=== Test: CI/CD Rollback Configuration ===" local deploy_wf=".github/workflows/deploy.yml" assert_contains "Rollback job defined" "$deploy_wf" "rollback:" assert_contains "Health check triggers rollback" "$deploy_wf" "needs.health-check.result" assert_contains "ECS --rollback flag used" "$deploy_wf" "--rollback" for svc in api darkwatch spamshield voiceprint; do assert_contains "Service '$svc' in deploy matrix" "$deploy_wf" "$svc" done } # ─── Test: Health Check Configuration ──────────────────────────── test_health_checks() { log "=== Test: Health Check Configuration ===" assert_contains "Container health check in ECS" "infra/modules/ecs/main.tf" "healthCheck" assert_contains "ALB health check defined" "infra/modules/ecs/main.tf" "health_check" assert_contains "ALB 5xx alarm configured" "infra/modules/cloudwatch/main.tf" "HTTPCode_Elb_5XX_Count" } # ─── Test: README References ───────────────────────────────────── test_readme() { log "=== Test: README References ===" assert_contains "README references ROLLBACK.md" "infra/README.md" "ROLLBACK.md" assert_contains "README documents rollback.sh" "infra/README.md" "rollback.sh" assert_contains "README documents rollback-compose.sh" "infra/README.md" "rollback-compose.sh" assert_contains "README documents rollback-migration.sh" "infra/README.md" "rollback-migration.sh" } # ─── Main ──────────────────────────────────────────────────────── main() { log "=== ShieldAI Rollback Test Suite ===" log "Suite: $TEST_SUITE" log "" case "$TEST_SUITE" in ecs|all) test_rollback_script test_cicd_rollback test_health_checks ;; compose|all) test_compose_script ;; migration) log "=== Test: Migration Rollback ===" assert_script_syntax "rollback-migration.sh syntax" "infra/scripts/rollback-migration.sh" assert_contains "Uses Secrets Manager" "infra/scripts/rollback-migration.sh" "secretsmanager" assert_contains "Uses drizzle-kit" "infra/scripts/rollback-migration.sh" "drizzle-kit" ;; esac test_file_structure test_script_syntax test_documentation test_readme log "" log "=== Results ===" log "Passed: $PASS" log "Failed: $FAIL" log "" if [[ $FAIL -gt 0 ]]; then log "❌ SOME TESTS FAILED" return 1 fi log "✅ ALL TESTS PASSED" return 0 } main "$@"