From 26d9f8b050969dfaa2c9dfb714a872160b7db382 Mon Sep 17 00:00:00 2001 From: Michael Freno Date: Thu, 28 May 2026 08:59:24 -0400 Subject: [PATCH] clear references --- .env.prod.example | 3 +- .gitignore | 1 + README.md | 6 +- pnpm-lock.yaml | 33 +++++--- .../09-drizzle-schema-migration.md | 24 +++--- .../10-db-connection-migrations.md | 84 +++++++++---------- .../42-deployment-config.md | 16 ++-- tasks/kordant-unified-restructure/README.md | 2 +- .../06-db-connection-pooling.md | 15 ++-- tasks/web-production/15-docker-infra.md | 5 +- tasks/web-production/31-db-backup.md | 28 ++++--- web/package.json | 2 - 12 files changed, 113 insertions(+), 106 deletions(-) diff --git a/.env.prod.example b/.env.prod.example index cab6eaf..6f3e0a9 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -1,5 +1,6 @@ # Database -POSTGRES_PASSWORD=change_me_in_production +DATABASE_URL=libsql://your-db.turso.io +DATABASE_AUTH_TOKEN=your-token # API Keys HIBP_API_KEY="" diff --git a/.gitignore b/.gitignore index f8cc543..5b3055c 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ android/app/build # OS .DS_Store Thumbs.db +honker diff --git a/README.md b/README.md index 24eeaae..b2b665f 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Unified SolidStart monolith with tRPC, Drizzle ORM, and native mobile apps. └────────────────────────┼──────────────────────────────────────┘ │ ┌────────▼────────┐ - │ PostgreSQL │ + │ Turso (SQLite)│ │ + Redis │ └─────────────────┘ ``` @@ -79,7 +79,7 @@ kordant/ ├── docs/ # Brand guidelines, runbooks ├── scripts/ # Build and deployment scripts ├── tasks/ # Project task tracking -├── docker-compose.yml # Local dev (web + postgres + redis) +├── docker-compose.yml # Local dev (web + redis; DB is external Turso) ├── docker-compose.prod.yml # Production deployment └── .github/workflows/ # CI/CD pipelines ``` @@ -93,7 +93,7 @@ kordant/ | **Language** | TypeScript (Node.js ≥22) | | **Framework** | SolidStart (SSR + API server) | | **API** | tRPC (type-safe RPC) | -| **Database** | PostgreSQL 16 (Drizzle ORM) | +| **Database** | Turso / SQLite (Drizzle ORM) | | **Cache / Queue** | Redis 7 | | **Styling** | Tailwind CSS + CSS custom properties | | **Mobile iOS** | SwiftUI (native) | diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 85e012c..8fde119 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,9 +111,6 @@ importers: node-cron: specifier: ^4.2.1 version: 4.2.1 - pg: - specifier: ^8.21.0 - version: 8.21.0 pino: specifier: ^10.3.1 version: 10.3.1 @@ -157,9 +154,6 @@ importers: '@types/node-cron': specifier: ^3.0.11 version: 3.0.11 - '@types/pg': - specifier: ^8.20.0 - version: 8.20.0 '@types/ws': specifier: ^8.18.1 version: 8.18.1 @@ -6826,6 +6820,7 @@ snapshots: '@types/node': 25.9.1 pg-protocol: 1.14.0 pg-types: 2.2.0 + optional: true '@types/request@2.48.13': dependencies: @@ -8793,15 +8788,19 @@ snapshots: pg-cloudflare@1.4.0: optional: true - pg-connection-string@2.13.0: {} + pg-connection-string@2.13.0: + optional: true - pg-int8@1.0.1: {} + pg-int8@1.0.1: + optional: true pg-pool@3.14.0(pg@8.21.0): dependencies: pg: 8.21.0 + optional: true - pg-protocol@1.14.0: {} + pg-protocol@1.14.0: + optional: true pg-types@2.2.0: dependencies: @@ -8810,6 +8809,7 @@ snapshots: postgres-bytea: 1.0.1 postgres-date: 1.0.7 postgres-interval: 1.2.0 + optional: true pg@8.21.0: dependencies: @@ -8820,10 +8820,12 @@ snapshots: pgpass: 1.0.5 optionalDependencies: pg-cloudflare: 1.4.0 + optional: true pgpass@1.0.5: dependencies: split2: 4.2.0 + optional: true picocolors@1.1.1: {} @@ -8895,15 +8897,19 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - postgres-array@2.0.0: {} + postgres-array@2.0.0: + optional: true - postgres-bytea@1.0.1: {} + postgres-bytea@1.0.1: + optional: true - postgres-date@1.0.7: {} + postgres-date@1.0.7: + optional: true postgres-interval@1.2.0: dependencies: xtend: 4.0.2 + optional: true powershell-utils@0.1.0: {} @@ -9934,7 +9940,8 @@ snapshots: xmlchars@2.2.0: {} - xtend@4.0.2: {} + xtend@4.0.2: + optional: true y18n@5.0.8: {} diff --git a/tasks/kordant-unified-restructure/09-drizzle-schema-migration.md b/tasks/kordant-unified-restructure/09-drizzle-schema-migration.md index fe7fcc3..30025af 100644 --- a/tasks/kordant-unified-restructure/09-drizzle-schema-migration.md +++ b/tasks/kordant-unified-restructure/09-drizzle-schema-migration.md @@ -26,28 +26,28 @@ deliverables: - RemoveBrokers (InfoBroker, RemovalRequest, BrokerListing) - `web/src/server/db/schema/` — Optional split directory if single file becomes unwieldy: - `auth.ts`, `subscription.ts`, `darkwatch.ts`, `voiceprint.ts`, `spamshield.ts`, `alerts.ts`, `correlation.ts`, `reports.ts`, `marketing.ts`, `hometitle.ts`, `removebrokers.ts` -- All enums defined as TypeScript const arrays or Drizzle `pgEnum` +- All enums defined as TypeScript const arrays or Drizzle `enum()` - All indexes, unique constraints, and foreign keys preserved - Relations defined using Drizzle's `relations()` helper steps: 1. Read and analyze `packages/db/prisma/schema.prisma` completely. Document every model, enum, relation, index, and constraint. -2. Install Drizzle ORM and PostgreSQL driver in `web/`: +2. Install Drizzle ORM and Turso client in `web/`: - `drizzle-orm` - - `pg` (node-postgres) or `@neondatabase/serverless` if using Neon + - `@libsql/client` (Turso/libsql client) - `drizzle-kit` for migrations 3. Create `web/src/server/db/schema.ts` (or split directory). 4. For each Prisma model, create a Drizzle table definition: - Map Prisma field types to Drizzle column types: - - `String` → `varchar`, `text`, `uuid` + - `String` → `text`, `varchar` - `Int` → `integer` - `Float` → `real` - - `Boolean` → `boolean` - - `DateTime` → `timestamp` - - `Json` → `jsonb` - - `String[]` → `text().array()` + - `Boolean` → `integer` (SQLite has no native boolean, use 0/1) + - `DateTime` → `text` (ISO strings) or `integer` (Unix timestamp) + - `Json` → `text` (SQLite stores JSON as text) + - `String[]` → `text` (serialize to JSON string) - Preserve `@id`, `@default(uuid())`, `@unique`, `@index`, `@relation` - - Map Prisma enums to Drizzle `pgEnum()` + - Map Prisma enums to Drizzle `enum()` or `text()` with check constraints 5. Define all indexes using Drizzle's `.index()` and `.unique()` on table definitions. 6. Define relations using `relations()` helper for: - User → accounts, sessions, familyGroups, subscriptions, alerts, voice enrollments, etc. @@ -77,7 +77,7 @@ validation: - Run `cd web && npx drizzle-kit generate` and inspect generated SQL - Compare table count: Prisma schema has X models, Drizzle schema has X tables - Verify enum values match exactly between Prisma and Drizzle -- Run `npx drizzle-kit push` against a local PostgreSQL instance and confirm all tables are created +- Run `npx drizzle-kit push` against a Turso database (or local SQLite) and confirm all tables are created notes: - This is the most critical backend task. A missing field or incorrect relation will cascade into broken tRPC routers. @@ -85,4 +85,6 @@ notes: - Prisma's `@updatedAt` auto-timestamp can be replicated with Drizzle's `$onUpdateFn(() => new Date())`. - Keep the old Prisma schema file as reference until task 41. - Consider using `drizzle-zod` later (task 11+) to auto-generate validation schemas from Drizzle tables. -- The schema uses PostgreSQL-specific features (arrays, enums, jsonb). Ensure the Drizzle definitions use `pgTable`, `pgEnum`, etc. +- The schema uses SQLite-compatible types. Ensure the Drizzle definitions use `sqliteTable`, `enum()`, etc. +- SQLite stores JSON as text — serialize/deserialize in application code or use Drizzle's `json()` helper. +- SQLite doesn't have native array types — store arrays as JSON-encoded text strings. diff --git a/tasks/kordant-unified-restructure/10-db-connection-migrations.md b/tasks/kordant-unified-restructure/10-db-connection-migrations.md index 138f271..cf7ebec 100644 --- a/tasks/kordant-unified-restructure/10-db-connection-migrations.md +++ b/tasks/kordant-unified-restructure/10-db-connection-migrations.md @@ -1,25 +1,25 @@ -# 10. Database — PostgreSQL Connection, Migrations, and Seed Data +# 10. Database — Turso (SQLite) Connection, Migrations, and Seed Data meta: id: kordant-unified-restructure-10 feature: kordant-unified-restructure priority: P0 depends_on: [kordant-unified-restructure-09] - tags: [backend, database, drizzle, postgres] + tags: [backend, database, drizzle, turso, sqlite] objective: -- Set up the database connection layer, migration tooling, and seed data for the unified monolith. Replace the current SQLite setup in `web/` with PostgreSQL, matching the production database used by the legacy `packages/db`. +- Set up the database connection layer, migration tooling, and seed data for the unified monolith. Configure Turso (libsql/SQLite) as the database backend, matching the production database used by the legacy `packages/db`. deliverables: - `web/src/server/db/index.ts` — Database connection module: - - Creates PostgreSQL connection pool using `pg` or `@neondatabase/serverless` + - Creates Turso (libsql) client using `@libsql/client` - Exports `db` instance initialized with Drizzle ORM - - Handles connection string from environment variables - - Graceful shutdown hook to close pool + - Handles connection URL and auth token from environment variables + - Graceful shutdown hook to close client - `web/drizzle.config.ts` — Drizzle Kit configuration: - - Points to `src/server/db/schema.ts` - - Specifies PostgreSQL dialect - - Reads database URL from env + - Points to `src/server/db/schema/index.ts` + - Specifies `turso` dialect + - Reads database URL and auth token from env - `web/src/server/db/migrate.ts` — Migration runner script: - Programmatically runs pending migrations on startup - Can be called from `entry-server.tsx` or a standalone script @@ -28,35 +28,39 @@ deliverables: - Idempotent: can be run multiple times without duplicates - Useful for development and demo environments - Environment configuration: - - `DATABASE_URL` in `.env` and `.env.example` - - Connection pooling settings (max connections, timeout) + - `DATABASE_URL` and `DATABASE_AUTH_TOKEN` in `.env` and `.env.example` steps: 1. Install dependencies in `web/`: - `drizzle-orm`, `drizzle-kit` - - `pg` (for local/dev) or `@neondatabase/serverless` (for serverless deploy) - - `@types/pg` if using `pg` + - `@libsql/client` (Turso/libsql client) 2. Create `web/src/server/db/index.ts`: ```ts - import { drizzle } from 'drizzle-orm/node-postgres'; - import { Pool } from 'pg'; + import { createClient } from '@libsql/client'; + import { drizzle } from 'drizzle-orm/libsql'; import * as schema from './schema'; - const pool = new Pool({ connectionString: process.env.DATABASE_URL }); - export const db = drizzle(pool, { schema }); + const client = createClient({ + url: process.env.DATABASE_URL, + authToken: process.env.DATABASE_AUTH_TOKEN, + }); + export const db = drizzle(client, { schema }); ``` 3. Update `web/drizzle.config.ts`: ```ts import { defineConfig } from 'drizzle-kit'; export default defineConfig({ - schema: './src/server/db/schema.ts', + schema: './src/server/db/schema/index.ts', out: './drizzle', - dialect: 'postgresql', - dbCredentials: { url: process.env.DATABASE_URL! }, + dialect: 'turso', + dbCredentials: { + url: process.env.DATABASE_URL!, + authToken: process.env.DATABASE_AUTH_TOKEN, + }, }); ``` 4. Create `web/src/server/db/migrate.ts`: - - Import `migrate` from `drizzle-orm/node-postgres/migrator` + - Import `migrate` from `drizzle-orm/libsql/migrator` - Run migrations from `./drizzle` folder - Log success or error 5. Create `web/src/server/db/seed.ts`: @@ -64,50 +68,40 @@ steps: - Create: 2-3 users, 1 family group, 2 subscriptions (basic + premium), 3-5 watchlist items, 2-3 exposures, 5-10 alerts, 3 blog posts, 2 property watchlist items, 1 removal request - Use deterministic IDs or check for existing data before inserting 6. Add `db:migrate`, `db:generate`, `db:push`, `db:seed` scripts to `web/package.json`. -7. Create `.env.example` in `web/` with `DATABASE_URL=postgresql://...` +7. Create `.env.example` in `web/` with `DATABASE_URL=libsql://...` and `DATABASE_AUTH_TOKEN` 8. Test locally: - - Start PostgreSQL (Docker or local install) + - Create a Turso database (or use local SQLite with `libsql://./dev.db`) - Run `pnpm db:generate` to create migration SQL - Run `pnpm db:push` to apply schema - Run `pnpm db:seed` to populate data - - Verify tables exist with correct data using `psql` or a GUI tool + - Verify tables exist with correct data using `turso db shell` or a SQLite GUI tool 9. Update `web/src/server/db/schema.ts` from task 09 if any adjustments are needed based on migration output. steps: - Integration: `pnpm db:generate` produces valid SQL -- Integration: `pnpm db:push` creates all tables in local PostgreSQL +- Integration: `pnpm db:push` creates all tables in local Turso/SQLite - Integration: `pnpm db:seed` populates tables without errors - Integration: Application can query seeded data via `db.select()` acceptance_criteria: -- [ ] `web/src/server/db/index.ts` exports a working Drizzle `db` instance -- [ ] `drizzle.config.ts` is correctly configured for PostgreSQL +- [ ] `web/src/server/db/index.ts` exports a working Drizzle `db` instance with Turso client +- [ ] `drizzle.config.ts` is correctly configured for Turso (dialect: 'turso') - [ ] `pnpm db:generate` creates migration files in `web/drizzle/` -- [ ] `pnpm db:push` applies schema to a PostgreSQL database successfully +- [ ] `pnpm db:push` applies schema to a Turso database successfully - [ ] `pnpm db:seed` populates all relevant tables with sample data - [ ] The app can perform `db.select().from(users)` and return seeded users - [ ] Environment variables are documented in `.env.example` validation: - `cd web && pnpm db:generate` — check `drizzle/` folder for SQL files -- `cd web && pnpm db:push` — verify tables created via `psql -d kordant -c "\dt"` -- `cd web && pnpm db:seed` — verify data exists via `psql -d kordant -c "SELECT COUNT(*) FROM users;"` +- `cd web && pnpm db:push` — verify tables created via `turso db shell "SELECT name FROM sqlite_master WHERE type='table';"` +- `cd web && pnpm db:seed` — verify data exists via `turso db shell "SELECT COUNT(*) FROM users;"` - Create a temporary test route that queries `db.select().from(users)` and renders results notes: -- The legacy project used Prisma + PostgreSQL in production. We are keeping PostgreSQL but switching to Drizzle ORM. -- For local development, Docker Compose with PostgreSQL is recommended: - ```yaml - services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_USER: kordant - POSTGRES_PASSWORD: kordant - POSTGRES_DB: kordant - ports: - - "5432:5432" - ``` -- If deploying to Vercel/Netlify serverless, use `@neondatabase/serverless` instead of `pg`. +- We are using Turso (libsql/SQLite) as the database backend with Drizzle ORM. +- For local development, you can use a local SQLite file (`libsql://./dev.db`) or create a Turso database. +- Turso provides edge-distributed SQLite with fast read replicas — no connection pooling needed. - Migration files should be committed to git so all environments run the same migrations. -- The seed script should be idempotent. Use `ON CONFLICT DO NOTHING` or check existence before inserting. +- The seed script should be idempotent. Use `INSERT OR IGNORE` or check existence before inserting. +- SQLite doesn't support `DROP COLUMN` or `ALTER TABLE` extensively — plan schema changes carefully or use Turso's rewrap for migrations. diff --git a/tasks/kordant-unified-restructure/42-deployment-config.md b/tasks/kordant-unified-restructure/42-deployment-config.md index 5073c04..0967bc4 100644 --- a/tasks/kordant-unified-restructure/42-deployment-config.md +++ b/tasks/kordant-unified-restructure/42-deployment-config.md @@ -19,10 +19,10 @@ deliverables: - Non-root user for security - `docker-compose.yml` — Local development orchestration: - `web` service: builds from `web/Dockerfile`, ports `3000:3000` - - `postgres` service: PostgreSQL 16 with volume for data persistence - `redis` service: Redis 7 for job queues and caching - `nginx` service: reverse proxy with SSL termination (optional) - Environment variables from `.env` file + - Database is external (Turso) — no container needed - `docker-compose.prod.yml` — Production orchestration: - Similar to dev but with production-optimized settings - Volume mounts for uploads/logs @@ -55,7 +55,7 @@ deliverables: - Health check verification - Rollback on failure - `scripts/backup.sh` — Database backup script: - - `pg_dump` to timestamped file + - `turso db shell` or `sqlite3` dump to timestamped file - Upload to S3 or similar storage steps: @@ -87,14 +87,14 @@ steps: ``` 2. Create `docker-compose.yml`: - Define services with appropriate environment variables - - PostgreSQL with `volumes: postgres_data:/var/lib/postgresql/data` - Redis with `volumes: redis_data:/data` - Network configuration + - Database is external (Turso), configured via `DATABASE_URL` and `DATABASE_AUTH_TOKEN` 3. Create `docker-compose.prod.yml`: - Add restart policies: `unless-stopped` - Add resource limits: `mem_limit`, `cpus` - Add logging driver configuration - - Remove port bindings for internal services (postgres, redis) + - Remove port bindings for internal services (redis) 4. Create `.github/workflows/ci.yml`: - Trigger: push to any branch, pull requests - Jobs: @@ -118,13 +118,13 @@ steps: - Group by category (Database, Auth, Payments, APIs, etc.) 7. Create `scripts/deploy.sh`: - `#!/bin/bash` with error handling (`set -euo pipefail`) - - Backup database: `docker exec postgres pg_dump ...` + - Backup database: `turso db shell ".dump" > backup.sql` - Run migrations: `docker compose exec web pnpm db:migrate` - Deploy: `docker compose -f docker-compose.prod.yml up -d` - Health check: `curl -f http://localhost:3000/health` - Rollback on failure: `docker compose rollback` or restore backup 8. Create `scripts/backup.sh`: - - Generate timestamped dump + - Generate timestamped dump via Turso CLI or SQLite dump - Compress with gzip - Upload to S3 using AWS CLI or rclone - Retain last 30 backups @@ -145,7 +145,7 @@ steps: acceptance_criteria: - [ ] `web/Dockerfile` builds a production-ready container -- [ ] `docker-compose.yml` orchestrates web, postgres, and redis for local dev +- [ ] `docker-compose.yml` orchestrates web and redis for local dev (database is external Turso) - [ ] `docker-compose.prod.yml` is optimized for production with restart policies and resource limits - [ ] CI pipeline runs lint, type check, tests, build, and audit on every PR - [ ] CD pipeline builds and deploys on release tags @@ -168,7 +168,7 @@ validation: notes: - The unified monolith simplifies deployment significantly: one container instead of 5+ microservices. - For high availability, run multiple web container instances behind a load balancer (nginx, AWS ALB, etc.). -- Consider using a managed database (RDS, Supabase, Neon) instead of self-hosted PostgreSQL for production. +- Database is Turso (edge-distributed SQLite) — no container needed, accessed via `DATABASE_URL`. - For Redis, consider Upstash or ElastiCache for managed service. - The web app uses SolidStart with Nitro, which can run as a standalone server. Ensure the `.output/server/index.mjs` entry point is correct. - For SSL, use Let's Encrypt with nginx or a managed load balancer. Document certificate renewal. diff --git a/tasks/kordant-unified-restructure/README.md b/tasks/kordant-unified-restructure/README.md index b1fa551..026d091 100644 --- a/tasks/kordant-unified-restructure/README.md +++ b/tasks/kordant-unified-restructure/README.md @@ -14,7 +14,7 @@ Tasks - [x] 07 — Auth Pages — Login, Signup, Password Reset, Onboarding → `07-auth-pages.md` - [x] 08 — Migrate & Redesign Existing Pages — Blog, Ads, Dashboard Shell → `08-migrate-existing-pages.md` - [x] 09 — Database — Migrate Full Prisma Schema to Drizzle ORM → `09-drizzle-schema-migration.md` -- [x] 10 — Database — PostgreSQL Connection, Migrations, and Seed Data → `10-db-connection-migrations.md` +- [x] 10 — Database — Turso (SQLite) Connection, Migrations, and Seed Data → `10-db-connection-migrations.md` - [x] 11 — tRPC Foundation — Auth Context, Middleware, and Protected Procedures → `11-trpc-auth-context.md` - [x] 12 — Backend Router — User & Family Group Management → `12-user-family-router.md` - [x] 13 — Backend Router — Subscriptions, Billing, and Stripe Webhooks → `13-subscription-billing-router.md` diff --git a/tasks/web-production/06-db-connection-pooling.md b/tasks/web-production/06-db-connection-pooling.md index 4fecf85..83c3d4e 100644 --- a/tasks/web-production/06-db-connection-pooling.md +++ b/tasks/web-production/06-db-connection-pooling.md @@ -5,7 +5,7 @@ meta: feature: web-production priority: P1 depends_on: [] - tags: [performance, database, production] + tags: [performance, database, production, turso, sqlite] objective: - Optimize database connections and queries for production load @@ -17,10 +17,10 @@ deliverables: - Slow query logging steps: -1. Configure connection pooling: - - If using PostgreSQL: configure PgBouncer or use @libsql/client pooling - - Set max connections based on server instances (e.g., 20 per instance) - - Add connection timeout and idle timeout settings +1. Configure connection handling: + - Turso/libsql handles connection management internally — no external pool needed + - Configure `@libsql/client` with appropriate timeout settings + - Leverage Turso's edge distribution for low-latency reads 2. Audit all Drizzle queries for performance: - Check web/src/server/db/schema/*.ts for missing indexes - Review web/src/server/api/routers/*.ts for N+1 queries @@ -58,5 +58,6 @@ validation: notes: - Current schema has some indexes but may need more for production scale -- Drizzle ORM doesn't automatically handle connection pooling — configure at driver level -- Consider read replicas if dashboard load is heavy +- Turso/libsql handles connection management internally — no PgBouncer or connection pool needed +- Turso provides edge read replicas automatically — configure primary for writes, edges for reads +- SQLite has different query patterns than PostgreSQL — avoid heavy JOINs on large tables, prefer indexed lookups diff --git a/tasks/web-production/15-docker-infra.md b/tasks/web-production/15-docker-infra.md index 9ce0807..0fe60e3 100644 --- a/tasks/web-production/15-docker-infra.md +++ b/tasks/web-production/15-docker-infra.md @@ -30,7 +30,7 @@ steps: 3. Create docker-compose.prod.yml: - Web app service with replicas - Redis service with persistence - - PostgreSQL service (or external) + - Database is external (Turso) — no container needed - Nginx reverse proxy with SSL termination - Watchtower for automatic updates 4. Add security scanning: @@ -42,7 +42,7 @@ steps: - VPC, subnets, security groups - ECS/Fargate or Kubernetes deployment - Load balancer with SSL - - RDS/Cloud SQL for PostgreSQL + - Turso database (managed — no IaaS needed, configure via environment variables) - ElastiCache/Memorystore for Redis 6. Add environment-specific configs: - Production nginx.conf with rate limiting @@ -73,3 +73,4 @@ notes: - Current scheduler/Dockerfile copies many source files — optimize with .dockerignore - Consider using distroless images for even smaller footprint - Use AWS Fargate or Google Cloud Run for serverless containers +- Turso is fully managed — no database container or IaaS needed, just `DATABASE_URL` and `DATABASE_AUTH_TOKEN` diff --git a/tasks/web-production/31-db-backup.md b/tasks/web-production/31-db-backup.md index e3f5972..6ea5e8b 100644 --- a/tasks/web-production/31-db-backup.md +++ b/tasks/web-production/31-db-backup.md @@ -5,10 +5,10 @@ meta: feature: web-production priority: P1 depends_on: [] - tags: [database, reliability, production] + tags: [database, reliability, production, turso, sqlite] objective: -- Implement automated database backups with point-in-time recovery capability +- Implement automated database backups with point-in-time recovery capability using Turso's built-in backup features deliverables: - Automated daily backups @@ -18,24 +18,24 @@ deliverables: steps: 1. Set up automated backups: - - If PostgreSQL: configure pg_dump cron job or managed backups (RDS, Cloud SQL) - - If SQLite/Turso: configure Turso database branching/backups + - Configure Turso database backups using the Turso CLI (`turso db backup`) + - Use Turso's branching feature for safe schema changes and rollbacks - Daily full backups at off-peak hours (3 AM UTC) - - Hourly incremental backups (WAL archiving for Postgres) + - Leverage Turso's built-in replication for high availability 2. Configure backup storage: - - Store in separate region/cloud provider (S3, GCS, R2) + - Store backups in separate region/cloud provider (S3, GCS, R2) - Encrypt backups at rest - Versioning enabled (protect against deletion) 3. Implement point-in-time recovery: - - WAL archiving for PostgreSQL - - Transaction log backups every 15 minutes - - Test recovery to specific timestamp + - Use Turso branches to test migrations before applying to main + - Schedule regular backups to external storage + - Test recovery to specific backup point 4. Add backup monitoring: - Alert on backup failure - Track backup size and duration - Verify backup integrity (checksum) 5. Test restore procedures: - - Monthly restore test to staging environment + - Monthly restore test to staging environment using `turso db restore` - Document step-by-step restore process - Measure RTO (Recovery Time Objective) and RPO (Recovery Point Objective) - Target: RTO < 1 hour, RPO < 15 minutes @@ -55,7 +55,7 @@ tests: - Monitoring: Verify backup alerts acceptance_criteria: -- Daily automated backups running successfully +- Daily automated Turso backups running successfully - Backups stored in separate region with encryption - Point-in-time recovery tested and working - Backup failures trigger alerts within 5 minutes @@ -63,6 +63,7 @@ acceptance_criteria: - RTO < 1 hour, RPO < 15 minutes - Retention policy enforced automatically - Redis backups included in strategy +- Turso branching workflow documented for safe schema changes validation: - Check backup storage → daily backups present @@ -71,7 +72,8 @@ validation: - Check retention → old backups purged per policy notes: -- Turso offers automatic backups for SQLite — verify configuration -- RDS automated backups are easiest for PostgreSQL +- Turso offers automatic backups and branching for SQLite — leverage both +- Use `turso db backup ` for manual backups +- Use `turso branch create ` to create safe testing environments - Test restores are critical — untested backups are useless - Document restore process for on-call engineers diff --git a/web/package.json b/web/package.json index c82934c..8435532 100644 --- a/web/package.json +++ b/web/package.json @@ -34,7 +34,6 @@ "ioredis": "^5.10.1", "jose": "^5", "node-cron": "^4.2.1", - "pg": "^8.21.0", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "puppeteer": "^25.0.4", @@ -54,7 +53,6 @@ "devDependencies": { "@playwright/test": "^1.60.0", "@types/node-cron": "^3.0.11", - "@types/pg": "^8.20.0", "@types/ws": "^8.18.1", "drizzle-kit": "^0.31.10", "jsdom": "^29.1.1",