Add tier-based scan scheduler and webhook triggers (FRE-4498)
- ScanScheduler: tier-based scheduling (BASIC=24h, PLUS=6h, PREMIUM=1h) - WebhookHandler: HMAC-verified webhook ingestion with SCAN_TRIGGER support - API routes: /scheduler and /webhooks endpoints under /api/v1/darkwatch - Jobs: scheduled scan checker + webhook retry processor via BullMQ - Schema: ScanSchedule, WebhookEvent models; ScanJob.scheduledBy field - Types: ScheduleStatus, WebhookEventType, WebhookTriggerInput - Tests: scheduler lifecycle + webhook signature/processing tests Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
122
.github/workflows/ci.yml
vendored
Normal file
122
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,122 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
NODE_VERSION: "20"
|
||||
PNPM_VERSION: "9"
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: "npm"
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
- name: Run linter
|
||||
run: npm run lint
|
||||
|
||||
typecheck:
|
||||
name: Type Check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: "npm"
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
- name: Build all packages
|
||||
run: npm run build
|
||||
|
||||
test:
|
||||
name: Test Suite
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_DB: shieldai
|
||||
POSTGRES_USER: shieldai
|
||||
POSTGRES_PASSWORD: shieldai_dev
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U shieldai"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: "npm"
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
- name: Generate Prisma client
|
||||
run: npx prisma generate --schema=packages/db/prisma/schema.prisma
|
||||
env:
|
||||
DATABASE_URL: "postgresql://shieldai:shieldai_dev@localhost:5432/shieldai"
|
||||
- name: Run tests
|
||||
run: npm run test
|
||||
env:
|
||||
DATABASE_URL: "postgresql://shieldai:shieldai_dev@localhost:5432/shieldai"
|
||||
REDIS_URL: "redis://localhost:6379"
|
||||
|
||||
docker-build:
|
||||
name: Docker Build
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, typecheck]
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- name: api
|
||||
context: .
|
||||
dockerfile: packages/api/Dockerfile
|
||||
- name: darkwatch
|
||||
context: .
|
||||
dockerfile: services/darkwatch/Dockerfile
|
||||
- name: spamshield
|
||||
context: .
|
||||
dockerfile: services/spamshield/Dockerfile
|
||||
- name: voiceprint
|
||||
context: .
|
||||
dockerfile: services/voiceprint/Dockerfile
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ${{ matrix.context }}
|
||||
file: ${{ matrix.dockerfile }}
|
||||
push: false
|
||||
tags: shieldai-${{ matrix.name }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
101
.github/workflows/deploy.yml
vendored
Normal file
101
.github/workflows/deploy.yml
vendored
Normal file
@@ -0,0 +1,101 @@
|
||||
name: Deploy
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
concurrency:
|
||||
group: deploy-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
NODE_VERSION: "20"
|
||||
|
||||
jobs:
|
||||
detect-environment:
|
||||
name: Detect Environment
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
environment: ${{ steps.detect.outputs.environment }}
|
||||
steps:
|
||||
- name: Detect deployment target
|
||||
id: detect
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "release" ]; then
|
||||
echo "environment=production" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "environment=staging" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
build-and-push:
|
||||
name: Build and Push Docker Images
|
||||
runs-on: ubuntu-latest
|
||||
needs: detect-environment
|
||||
environment: ${{ needs.detect-environment.outputs.environment }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- name: api
|
||||
dockerfile: packages/api/Dockerfile
|
||||
- name: darkwatch
|
||||
dockerfile: services/darkwatch/Dockerfile
|
||||
- name: spamshield
|
||||
dockerfile: services/spamshield/Dockerfile
|
||||
- name: voiceprint
|
||||
dockerfile: services/voiceprint/Dockerfile
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Login to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Calculate image tag
|
||||
id: tag
|
||||
run: |
|
||||
if [ "${{ needs.detect-environment.outputs.environment }}" = "production" ]; then
|
||||
echo "tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag=staging-${{ github.sha }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Build and push ${{ matrix.name }}
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ${{ matrix.dockerfile }}
|
||||
push: true
|
||||
tags: ghcr.io/${{ github.repository_owner }}/shieldai-${{ matrix.name }}:${{ steps.tag.outputs.tag }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
deploy:
|
||||
name: Deploy to ${{ needs.detect-environment.outputs.environment }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: [detect-environment, build-and-push]
|
||||
environment: ${{ needs.detect-environment.outputs.environment }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Calculate deployment tag
|
||||
id: tag
|
||||
run: |
|
||||
if [ "${{ needs.detect-environment.outputs.environment }}" = "production" ]; then
|
||||
echo "tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag=staging-${{ github.sha }}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Deploy via Docker Compose
|
||||
uses: appleboy/ssh-action@v1
|
||||
with:
|
||||
host: ${{ secrets.DEPLOY_HOST }}
|
||||
username: ${{ secrets.DEPLOY_USER }}
|
||||
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
script: |
|
||||
cd /opt/shieldai
|
||||
export DOCKER_TAG="${{ steps.tag.outputs.tag }}"
|
||||
export ENVIRONMENT="${{ needs.detect-environment.outputs.environment }}"
|
||||
docker compose pull
|
||||
docker compose up -d
|
||||
docker image prune -f
|
||||
Reference in New Issue
Block a user