re-init
This commit is contained in:
11
.env.local.example
Normal file
11
.env.local.example
Normal file
@@ -0,0 +1,11 @@
|
||||
# Plant Disease Identification — Environment Variables
|
||||
# Copy this file to .env.local and fill in real values.
|
||||
|
||||
# Path to compiled ML model files (relative to public/)
|
||||
NEXT_PUBLIC_MODEL_PATH=/models
|
||||
|
||||
# (Future) API key for external plant disease databases
|
||||
# PLANT_ID_API_KEY=
|
||||
|
||||
# (Future) OpenAI / vision model API key for image analysis
|
||||
# OPENAI_API_KEY=
|
||||
2
.gitattributes
vendored
Normal file
2
.gitattributes
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
# Git LFS for compiled ML model files
|
||||
public/models/* filter=lfs diff=lfs merge=lfs -text
|
||||
103
.github/workflows/ci.yml
vendored
Normal file
103
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,103 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: apps/web
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: apps/web/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run ESLint
|
||||
run: npm run lint
|
||||
|
||||
typecheck:
|
||||
name: Type Check
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: apps/web
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: apps/web/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run TypeScript check
|
||||
run: npx tsc --noEmit
|
||||
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: apps/web
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: apps/web/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run unit tests
|
||||
run: npx vitest run --reporter=verbose --coverage
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, typecheck, test]
|
||||
defaults:
|
||||
run:
|
||||
working-directory: apps/web
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
cache-dependency-path: apps/web/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build Next.js app
|
||||
run: npm run build
|
||||
env:
|
||||
NODE_ENV: production
|
||||
50
.gitignore
vendored
Normal file
50
.gitignore
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/versions
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# next.js
|
||||
/.next/
|
||||
/out/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files — commit templates, not actual secrets
|
||||
.env*
|
||||
!.env.local.example
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
|
||||
# Uploaded images (user-generated content)
|
||||
/public/uploads/*
|
||||
!/public/uploads/.gitkeep
|
||||
|
||||
# ML model binaries (tracked via Git LFS)
|
||||
# Add to .gitattributes for LFS:
|
||||
# public/models/* filter=lfs diff=lfs merge=lfs -text
|
||||
8
.prettierrc
Normal file
8
.prettierrc
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"semi": true,
|
||||
"trailingComma": "all",
|
||||
"singleQuote": false,
|
||||
"printWidth": 100,
|
||||
"tabWidth": 2,
|
||||
"useTabs": false
|
||||
}
|
||||
25
.vercelignore
Normal file
25
.vercelignore
Normal file
@@ -0,0 +1,25 @@
|
||||
# Next.js build output (Vercel rebuilds)
|
||||
.next/
|
||||
|
||||
# Dependencies (Vercel installs these)
|
||||
node_modules/
|
||||
|
||||
# Python venv and caches
|
||||
.tfjs-venv/
|
||||
.ruff_cache/
|
||||
|
||||
# Data files (large ML datasets — 37G)
|
||||
data/
|
||||
|
||||
# Test coverage
|
||||
coverage/
|
||||
|
||||
# Git (Vercel prefers no git dir for CLI deploys)
|
||||
.git/
|
||||
|
||||
# Scripts and task files (build-time only)
|
||||
scripts/
|
||||
tasks/
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
5
AGENTS.md
Normal file
5
AGENTS.md
Normal file
@@ -0,0 +1,5 @@
|
||||
<!-- BEGIN:nextjs-agent-rules -->
|
||||
# This is NOT the Next.js you know
|
||||
|
||||
This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` before writing any code. Heed deprecation notices.
|
||||
<!-- END:nextjs-agent-rules -->
|
||||
36
README.md
Normal file
36
README.md
Normal file
@@ -0,0 +1,36 @@
|
||||
This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app).
|
||||
|
||||
## Getting Started
|
||||
|
||||
First, run the development server:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
# or
|
||||
yarn dev
|
||||
# or
|
||||
pnpm dev
|
||||
# or
|
||||
bun dev
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
|
||||
|
||||
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
|
||||
|
||||
This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
|
||||
|
||||
## Learn More
|
||||
|
||||
To learn more about Next.js, take a look at the following resources:
|
||||
|
||||
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
|
||||
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
|
||||
|
||||
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
|
||||
|
||||
## Deploy on Vercel
|
||||
|
||||
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
|
||||
|
||||
Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.
|
||||
1
data/.gitignore
vendored
Normal file
1
data/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
dataset
|
||||
11
drizzle.config.ts
Normal file
11
drizzle.config.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import { defineConfig } from "drizzle-kit";
|
||||
|
||||
export default defineConfig({
|
||||
schema: "./src/lib/db/schema.ts",
|
||||
out: "./drizzle",
|
||||
dialect: "turso",
|
||||
dbCredentials: {
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
},
|
||||
});
|
||||
46
drizzle/0000_flippant_talon.sql
Normal file
46
drizzle/0000_flippant_talon.sql
Normal file
@@ -0,0 +1,46 @@
|
||||
CREATE TABLE `diseases` (
|
||||
`id` text PRIMARY KEY NOT NULL,
|
||||
`plant_id` text NOT NULL,
|
||||
`name` text NOT NULL,
|
||||
`scientific_name` text DEFAULT '' NOT NULL,
|
||||
`causal_agent_type` text NOT NULL,
|
||||
`description` text DEFAULT '' NOT NULL,
|
||||
`symptoms` text DEFAULT '[]' NOT NULL,
|
||||
`causes` text DEFAULT '[]' NOT NULL,
|
||||
`treatment` text DEFAULT '[]' NOT NULL,
|
||||
`prevention` text DEFAULT '[]' NOT NULL,
|
||||
`lookalike_ids` text DEFAULT '[]' NOT NULL,
|
||||
`severity` text NOT NULL,
|
||||
`source_url` text DEFAULT '' NOT NULL,
|
||||
`created_at` text DEFAULT (datetime('now')) NOT NULL,
|
||||
`updated_at` text DEFAULT (datetime('now')) NOT NULL,
|
||||
FOREIGN KEY (`plant_id`) REFERENCES `plants`(`id`) ON UPDATE no action ON DELETE no action
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `idx_diseases_plant_id` ON `diseases` (`plant_id`);--> statement-breakpoint
|
||||
CREATE INDEX `idx_diseases_causal_agent` ON `diseases` (`causal_agent_type`);--> statement-breakpoint
|
||||
CREATE INDEX `idx_diseases_severity` ON `diseases` (`severity`);--> statement-breakpoint
|
||||
CREATE TABLE `plants` (
|
||||
`id` text PRIMARY KEY NOT NULL,
|
||||
`common_name` text NOT NULL,
|
||||
`scientific_name` text NOT NULL,
|
||||
`family` text NOT NULL,
|
||||
`category` text NOT NULL,
|
||||
`care_summary` text DEFAULT '' NOT NULL,
|
||||
`image_url` text DEFAULT '' NOT NULL,
|
||||
`created_at` text DEFAULT (datetime('now')) NOT NULL,
|
||||
`updated_at` text DEFAULT (datetime('now')) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `idx_plants_category` ON `plants` (`category`);--> statement-breakpoint
|
||||
CREATE INDEX `idx_plants_common_name` ON `plants` (`common_name`);--> statement-breakpoint
|
||||
CREATE TABLE `scrape_sources` (
|
||||
`id` text PRIMARY KEY NOT NULL,
|
||||
`source_type` text NOT NULL,
|
||||
`source_url` text NOT NULL,
|
||||
`last_scraped_at` text,
|
||||
`entries_count` integer DEFAULT 0,
|
||||
`status` text DEFAULT 'pending' NOT NULL,
|
||||
`error_message` text,
|
||||
`created_at` text DEFAULT (datetime('now')) NOT NULL
|
||||
);
|
||||
1
drizzle/0001_add-disease-images.sql
Normal file
1
drizzle/0001_add-disease-images.sql
Normal file
@@ -0,0 +1 @@
|
||||
ALTER TABLE `diseases` ADD `image_url` text DEFAULT '' NOT NULL;
|
||||
2
drizzle/0002_add-prevalence.sql
Normal file
2
drizzle/0002_add-prevalence.sql
Normal file
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE `diseases` ADD `prevalence` text DEFAULT 'uncommon' NOT NULL;--> statement-breakpoint
|
||||
CREATE INDEX `idx_diseases_prevalence` ON `diseases` (`prevalence`);
|
||||
7
drizzle/0003_giant_toad.sql
Normal file
7
drizzle/0003_giant_toad.sql
Normal file
@@ -0,0 +1,7 @@
|
||||
CREATE TABLE `plant_views` (
|
||||
`plant_id` text PRIMARY KEY NOT NULL,
|
||||
`view_count` integer DEFAULT 0 NOT NULL,
|
||||
FOREIGN KEY (`plant_id`) REFERENCES `plants`(`id`) ON UPDATE no action ON DELETE no action
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `idx_plant_views_count` ON `plant_views` (`view_count`);
|
||||
14
drizzle/0004_add-flagged-content.sql
Normal file
14
drizzle/0004_add-flagged-content.sql
Normal file
@@ -0,0 +1,14 @@
|
||||
CREATE TABLE `flagged_content` (
|
||||
`id` text PRIMARY KEY NOT NULL,
|
||||
`content_type` text NOT NULL,
|
||||
`content_id` text NOT NULL,
|
||||
`field_name` text NOT NULL,
|
||||
`notes` text DEFAULT '',
|
||||
`flag_count` integer DEFAULT 1 NOT NULL,
|
||||
`created_at` text DEFAULT (datetime('now')) NOT NULL,
|
||||
`updated_at` text DEFAULT (datetime('now')) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `idx_flagged_content_type` ON `flagged_content` (`content_type`);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX `idx_flagged_content_id` ON `flagged_content` (`content_id`);
|
||||
1
drizzle/0005_add-prevalence-score.sql
Normal file
1
drizzle/0005_add-prevalence-score.sql
Normal file
@@ -0,0 +1 @@
|
||||
ALTER TABLE `diseases` ADD COLUMN `prevalence_score` integer DEFAULT 0 NOT NULL;
|
||||
340
drizzle/meta/0000_snapshot.json
Normal file
340
drizzle/meta/0000_snapshot.json
Normal file
@@ -0,0 +1,340 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "5471dc75-3736-4b26-b7a9-0629c9b1efa0",
|
||||
"prevId": "00000000-0000-0000-0000-000000000000",
|
||||
"tables": {
|
||||
"diseases": {
|
||||
"name": "diseases",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"plant_id": {
|
||||
"name": "plant_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"name": {
|
||||
"name": "name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"causal_agent_type": {
|
||||
"name": "causal_agent_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"description": {
|
||||
"name": "description",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"symptoms": {
|
||||
"name": "symptoms",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"causes": {
|
||||
"name": "causes",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"treatment": {
|
||||
"name": "treatment",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"prevention": {
|
||||
"name": "prevention",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"lookalike_ids": {
|
||||
"name": "lookalike_ids",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"severity": {
|
||||
"name": "severity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_diseases_plant_id": {
|
||||
"name": "idx_diseases_plant_id",
|
||||
"columns": [
|
||||
"plant_id"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_causal_agent": {
|
||||
"name": "idx_diseases_causal_agent",
|
||||
"columns": [
|
||||
"causal_agent_type"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_severity": {
|
||||
"name": "idx_diseases_severity",
|
||||
"columns": [
|
||||
"severity"
|
||||
],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"diseases_plant_id_plants_id_fk": {
|
||||
"name": "diseases_plant_id_plants_id_fk",
|
||||
"tableFrom": "diseases",
|
||||
"tableTo": "plants",
|
||||
"columnsFrom": [
|
||||
"plant_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"plants": {
|
||||
"name": "plants",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"common_name": {
|
||||
"name": "common_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"family": {
|
||||
"name": "family",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"category": {
|
||||
"name": "category",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"care_summary": {
|
||||
"name": "care_summary",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_plants_category": {
|
||||
"name": "idx_plants_category",
|
||||
"columns": [
|
||||
"category"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_plants_common_name": {
|
||||
"name": "idx_plants_common_name",
|
||||
"columns": [
|
||||
"common_name"
|
||||
],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"scrape_sources": {
|
||||
"name": "scrape_sources",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_type": {
|
||||
"name": "source_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"last_scraped_at": {
|
||||
"name": "last_scraped_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"entries_count": {
|
||||
"name": "entries_count",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": 0
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'pending'"
|
||||
},
|
||||
"error_message": {
|
||||
"name": "error_message",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
"enums": {},
|
||||
"_meta": {
|
||||
"schemas": {},
|
||||
"tables": {},
|
||||
"columns": {}
|
||||
},
|
||||
"internal": {
|
||||
"indexes": {}
|
||||
}
|
||||
}
|
||||
348
drizzle/meta/0001_snapshot.json
Normal file
348
drizzle/meta/0001_snapshot.json
Normal file
@@ -0,0 +1,348 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "6f2de82b-c1f9-42de-b03c-1c1f0c02b7c9",
|
||||
"prevId": "5471dc75-3736-4b26-b7a9-0629c9b1efa0",
|
||||
"tables": {
|
||||
"diseases": {
|
||||
"name": "diseases",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"plant_id": {
|
||||
"name": "plant_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"name": {
|
||||
"name": "name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"causal_agent_type": {
|
||||
"name": "causal_agent_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"description": {
|
||||
"name": "description",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"symptoms": {
|
||||
"name": "symptoms",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"causes": {
|
||||
"name": "causes",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"treatment": {
|
||||
"name": "treatment",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"prevention": {
|
||||
"name": "prevention",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"lookalike_ids": {
|
||||
"name": "lookalike_ids",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"severity": {
|
||||
"name": "severity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_diseases_plant_id": {
|
||||
"name": "idx_diseases_plant_id",
|
||||
"columns": [
|
||||
"plant_id"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_causal_agent": {
|
||||
"name": "idx_diseases_causal_agent",
|
||||
"columns": [
|
||||
"causal_agent_type"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_severity": {
|
||||
"name": "idx_diseases_severity",
|
||||
"columns": [
|
||||
"severity"
|
||||
],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"diseases_plant_id_plants_id_fk": {
|
||||
"name": "diseases_plant_id_plants_id_fk",
|
||||
"tableFrom": "diseases",
|
||||
"tableTo": "plants",
|
||||
"columnsFrom": [
|
||||
"plant_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"plants": {
|
||||
"name": "plants",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"common_name": {
|
||||
"name": "common_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"family": {
|
||||
"name": "family",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"category": {
|
||||
"name": "category",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"care_summary": {
|
||||
"name": "care_summary",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_plants_category": {
|
||||
"name": "idx_plants_category",
|
||||
"columns": [
|
||||
"category"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_plants_common_name": {
|
||||
"name": "idx_plants_common_name",
|
||||
"columns": [
|
||||
"common_name"
|
||||
],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"scrape_sources": {
|
||||
"name": "scrape_sources",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_type": {
|
||||
"name": "source_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"last_scraped_at": {
|
||||
"name": "last_scraped_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"entries_count": {
|
||||
"name": "entries_count",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": 0
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'pending'"
|
||||
},
|
||||
"error_message": {
|
||||
"name": "error_message",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
"enums": {},
|
||||
"_meta": {
|
||||
"schemas": {},
|
||||
"tables": {},
|
||||
"columns": {}
|
||||
},
|
||||
"internal": {
|
||||
"indexes": {}
|
||||
}
|
||||
}
|
||||
347
drizzle/meta/0002_snapshot.json
Normal file
347
drizzle/meta/0002_snapshot.json
Normal file
@@ -0,0 +1,347 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "7a3efb1c-d2e3-4f56-a789-0b1234567890",
|
||||
"prevId": "6f2de82b-c1f9-42de-b03c-1c1f0c02b7c9",
|
||||
"tables": {
|
||||
"diseases": {
|
||||
"name": "diseases",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"plant_id": {
|
||||
"name": "plant_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"name": {
|
||||
"name": "name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"causal_agent_type": {
|
||||
"name": "causal_agent_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"description": {
|
||||
"name": "description",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"symptoms": {
|
||||
"name": "symptoms",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"causes": {
|
||||
"name": "causes",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"treatment": {
|
||||
"name": "treatment",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"prevention": {
|
||||
"name": "prevention",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"lookalike_ids": {
|
||||
"name": "lookalike_ids",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"severity": {
|
||||
"name": "severity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"prevalence": {
|
||||
"name": "prevalence",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'uncommon'"
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_diseases_plant_id": {
|
||||
"name": "idx_diseases_plant_id",
|
||||
"columns": ["plant_id"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_causal_agent": {
|
||||
"name": "idx_diseases_causal_agent",
|
||||
"columns": ["causal_agent_type"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_severity": {
|
||||
"name": "idx_diseases_severity",
|
||||
"columns": ["severity"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_prevalence": {
|
||||
"name": "idx_diseases_prevalence",
|
||||
"columns": ["prevalence"],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"diseases_plant_id_plants_id_fk": {
|
||||
"name": "diseases_plant_id_plants_id_fk",
|
||||
"tableFrom": "diseases",
|
||||
"tableTo": "plants",
|
||||
"columnsFrom": ["plant_id"],
|
||||
"columnsTo": ["id"],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"plants": {
|
||||
"name": "plants",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"common_name": {
|
||||
"name": "common_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"family": {
|
||||
"name": "family",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"category": {
|
||||
"name": "category",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"care_summary": {
|
||||
"name": "care_summary",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_plants_category": {
|
||||
"name": "idx_plants_category",
|
||||
"columns": ["category"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_plants_common_name": {
|
||||
"name": "idx_plants_common_name",
|
||||
"columns": ["common_name"],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"scrape_sources": {
|
||||
"name": "scrape_sources",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_type": {
|
||||
"name": "source_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"last_scraped_at": {
|
||||
"name": "last_scraped_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"entries_count": {
|
||||
"name": "entries_count",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": 0
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'pending'"
|
||||
},
|
||||
"error_message": {
|
||||
"name": "error_message",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
"enums": {},
|
||||
"_meta": {
|
||||
"schemas": {},
|
||||
"tables": {},
|
||||
"columns": {}
|
||||
},
|
||||
"internal": {
|
||||
"indexes": {}
|
||||
}
|
||||
}
|
||||
410
drizzle/meta/0003_snapshot.json
Normal file
410
drizzle/meta/0003_snapshot.json
Normal file
@@ -0,0 +1,410 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "04ff83bd-e207-44d3-b8b7-8f82157bbeb8",
|
||||
"prevId": "7a3efb1c-d2e3-4f56-a789-0b1234567890",
|
||||
"tables": {
|
||||
"diseases": {
|
||||
"name": "diseases",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"plant_id": {
|
||||
"name": "plant_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"name": {
|
||||
"name": "name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"causal_agent_type": {
|
||||
"name": "causal_agent_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"description": {
|
||||
"name": "description",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"symptoms": {
|
||||
"name": "symptoms",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"causes": {
|
||||
"name": "causes",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"treatment": {
|
||||
"name": "treatment",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"prevention": {
|
||||
"name": "prevention",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"lookalike_ids": {
|
||||
"name": "lookalike_ids",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"prevalence": {
|
||||
"name": "prevalence",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'uncommon'"
|
||||
},
|
||||
"severity": {
|
||||
"name": "severity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_diseases_plant_id": {
|
||||
"name": "idx_diseases_plant_id",
|
||||
"columns": [
|
||||
"plant_id"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_causal_agent": {
|
||||
"name": "idx_diseases_causal_agent",
|
||||
"columns": [
|
||||
"causal_agent_type"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_severity": {
|
||||
"name": "idx_diseases_severity",
|
||||
"columns": [
|
||||
"severity"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_prevalence": {
|
||||
"name": "idx_diseases_prevalence",
|
||||
"columns": [
|
||||
"prevalence"
|
||||
],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"diseases_plant_id_plants_id_fk": {
|
||||
"name": "diseases_plant_id_plants_id_fk",
|
||||
"tableFrom": "diseases",
|
||||
"tableTo": "plants",
|
||||
"columnsFrom": [
|
||||
"plant_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"plant_views": {
|
||||
"name": "plant_views",
|
||||
"columns": {
|
||||
"plant_id": {
|
||||
"name": "plant_id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"view_count": {
|
||||
"name": "view_count",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": 0
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_plant_views_count": {
|
||||
"name": "idx_plant_views_count",
|
||||
"columns": [
|
||||
"view_count"
|
||||
],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"plant_views_plant_id_plants_id_fk": {
|
||||
"name": "plant_views_plant_id_plants_id_fk",
|
||||
"tableFrom": "plant_views",
|
||||
"tableTo": "plants",
|
||||
"columnsFrom": [
|
||||
"plant_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"plants": {
|
||||
"name": "plants",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"common_name": {
|
||||
"name": "common_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"family": {
|
||||
"name": "family",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"category": {
|
||||
"name": "category",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"care_summary": {
|
||||
"name": "care_summary",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_plants_category": {
|
||||
"name": "idx_plants_category",
|
||||
"columns": [
|
||||
"category"
|
||||
],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_plants_common_name": {
|
||||
"name": "idx_plants_common_name",
|
||||
"columns": [
|
||||
"common_name"
|
||||
],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"scrape_sources": {
|
||||
"name": "scrape_sources",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_type": {
|
||||
"name": "source_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"last_scraped_at": {
|
||||
"name": "last_scraped_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"entries_count": {
|
||||
"name": "entries_count",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": 0
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'pending'"
|
||||
},
|
||||
"error_message": {
|
||||
"name": "error_message",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
"enums": {},
|
||||
"_meta": {
|
||||
"schemas": {},
|
||||
"tables": {},
|
||||
"columns": {}
|
||||
},
|
||||
"internal": {
|
||||
"indexes": {}
|
||||
}
|
||||
}
|
||||
469
drizzle/meta/0004_snapshot.json
Normal file
469
drizzle/meta/0004_snapshot.json
Normal file
@@ -0,0 +1,469 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "04ff83bd-e207-44d3-b8b7-8f82157bbeb9",
|
||||
"prevId": "04ff83bd-e207-44d3-b8b7-8f82157bbeb8",
|
||||
"tables": {
|
||||
"diseases": {
|
||||
"name": "diseases",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"plant_id": {
|
||||
"name": "plant_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"name": {
|
||||
"name": "name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"causal_agent_type": {
|
||||
"name": "causal_agent_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"description": {
|
||||
"name": "description",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"symptoms": {
|
||||
"name": "symptoms",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"causes": {
|
||||
"name": "causes",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"treatment": {
|
||||
"name": "treatment",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"prevention": {
|
||||
"name": "prevention",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"lookalike_ids": {
|
||||
"name": "lookalike_ids",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"prevalence": {
|
||||
"name": "prevalence",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'uncommon'"
|
||||
},
|
||||
"severity": {
|
||||
"name": "severity",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_diseases_plant_id": {
|
||||
"name": "idx_diseases_plant_id",
|
||||
"columns": ["plant_id"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_causal_agent": {
|
||||
"name": "idx_diseases_causal_agent",
|
||||
"columns": ["causal_agent_type"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_severity": {
|
||||
"name": "idx_diseases_severity",
|
||||
"columns": ["severity"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_diseases_prevalence": {
|
||||
"name": "idx_diseases_prevalence",
|
||||
"columns": ["prevalence"],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"diseases_plant_id_plants_id_fk": {
|
||||
"name": "diseases_plant_id_plants_id_fk",
|
||||
"tableFrom": "diseases",
|
||||
"tableTo": "plants",
|
||||
"columnsFrom": ["plant_id"],
|
||||
"columnsTo": ["id"],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"flagged_content": {
|
||||
"name": "flagged_content",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"content_type": {
|
||||
"name": "content_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"content_id": {
|
||||
"name": "content_id",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"field_name": {
|
||||
"name": "field_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"notes": {
|
||||
"name": "notes",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"flag_count": {
|
||||
"name": "flag_count",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": 1
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_flagged_content_type": {
|
||||
"name": "idx_flagged_content_type",
|
||||
"columns": ["content_type"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_flagged_content_id": {
|
||||
"name": "idx_flagged_content_id",
|
||||
"columns": ["content_id"],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"plant_views": {
|
||||
"name": "plant_views",
|
||||
"columns": {
|
||||
"plant_id": {
|
||||
"name": "plant_id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"view_count": {
|
||||
"name": "view_count",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": 0
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_plant_views_count": {
|
||||
"name": "idx_plant_views_count",
|
||||
"columns": ["view_count"],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"plant_views_plant_id_plants_id_fk": {
|
||||
"name": "plant_views_plant_id_plants_id_fk",
|
||||
"tableFrom": "plant_views",
|
||||
"tableTo": "plants",
|
||||
"columnsFrom": ["plant_id"],
|
||||
"columnsTo": ["id"],
|
||||
"onDelete": "no action",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"plants": {
|
||||
"name": "plants",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"common_name": {
|
||||
"name": "common_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"scientific_name": {
|
||||
"name": "scientific_name",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"family": {
|
||||
"name": "family",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"category": {
|
||||
"name": "category",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"care_summary": {
|
||||
"name": "care_summary",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"image_url": {
|
||||
"name": "image_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "''"
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"idx_plants_category": {
|
||||
"name": "idx_plants_category",
|
||||
"columns": ["category"],
|
||||
"isUnique": false
|
||||
},
|
||||
"idx_plants_common_name": {
|
||||
"name": "idx_plants_common_name",
|
||||
"columns": ["common_name"],
|
||||
"isUnique": false
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"scrape_sources": {
|
||||
"name": "scrape_sources",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_type": {
|
||||
"name": "source_type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"source_url": {
|
||||
"name": "source_url",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"last_scraped_at": {
|
||||
"name": "last_scraped_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"entries_count": {
|
||||
"name": "entries_count",
|
||||
"type": "integer",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": 0
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "'pending'"
|
||||
},
|
||||
"error_message": {
|
||||
"name": "error_message",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "(datetime('now'))"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
"enums": {},
|
||||
"_meta": {
|
||||
"schemas": {},
|
||||
"tables": {},
|
||||
"columns": {}
|
||||
},
|
||||
"internal": {
|
||||
"indexes": {}
|
||||
}
|
||||
}
|
||||
48
drizzle/meta/_journal.json
Normal file
48
drizzle/meta/_journal.json
Normal file
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "sqlite",
|
||||
"entries": [
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "6",
|
||||
"when": 1780704072268,
|
||||
"tag": "0000_flippant_talon",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 1,
|
||||
"version": "6",
|
||||
"when": 1780710023177,
|
||||
"tag": "0001_add-disease-images",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 2,
|
||||
"version": "6",
|
||||
"when": 1749268800000,
|
||||
"tag": "0002_add-prevalence",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 3,
|
||||
"version": "6",
|
||||
"when": 1749268800000,
|
||||
"tag": "0003_giant_toad",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 4,
|
||||
"version": "6",
|
||||
"when": 1751846400000,
|
||||
"tag": "0004_add-flagged-content",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 5,
|
||||
"version": "6",
|
||||
"when": 1751846400000,
|
||||
"tag": "0005_add-prevalence-score",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
18
eslint.config.mjs
Normal file
18
eslint.config.mjs
Normal file
@@ -0,0 +1,18 @@
|
||||
import { defineConfig, globalIgnores } from "eslint/config";
|
||||
import nextVitals from "eslint-config-next/core-web-vitals";
|
||||
import nextTs from "eslint-config-next/typescript";
|
||||
|
||||
const eslintConfig = defineConfig([
|
||||
...nextVitals,
|
||||
...nextTs,
|
||||
// Override default ignores of eslint-config-next.
|
||||
globalIgnores([
|
||||
// Default ignores of eslint-config-next:
|
||||
".next/**",
|
||||
"out/**",
|
||||
"build/**",
|
||||
"next-env.d.ts",
|
||||
]),
|
||||
]);
|
||||
|
||||
export default eslintConfig;
|
||||
39
next.config.ts
Normal file
39
next.config.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import type { NextConfig } from "next";
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
// Allow remote images from Wikimedia Commons
|
||||
images: {
|
||||
remotePatterns: [
|
||||
{
|
||||
protocol: "https",
|
||||
hostname: "upload.wikimedia.org",
|
||||
port: "",
|
||||
pathname: "/wikipedia/commons/**",
|
||||
search: "",
|
||||
},
|
||||
],
|
||||
},
|
||||
// Turbopack config (Next.js 16 default)
|
||||
turbopack: {
|
||||
resolveAlias: {
|
||||
// Optional ML backends — not installed, dynamic import fallback to mock
|
||||
"@tensorflow/tfjs": "./src/stubs/empty.ts",
|
||||
"@tensorflow/tfjs-node": "./src/stubs/empty.ts",
|
||||
"onnxruntime-node": "./src/stubs/empty.ts",
|
||||
},
|
||||
},
|
||||
// Webpack config (fallback)
|
||||
webpack: (config) => {
|
||||
config.resolve.alias = {
|
||||
...config.resolve.alias,
|
||||
sharp: false,
|
||||
"detect-libc": false,
|
||||
"@tensorflow/tfjs": false,
|
||||
"@tensorflow/tfjs-node": false,
|
||||
"onnxruntime-node": false,
|
||||
};
|
||||
return config;
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
11366
package-lock.json
generated
Normal file
11366
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
48
package.json
Normal file
48
package.json
Normal file
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"name": "plant-disease-id",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "eslint",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"flagged-report": "npx tsx scripts/generate-flagged-report.ts",
|
||||
"flagged-report:all": "npx tsx scripts/generate-flagged-report.ts --min-flags=1",
|
||||
"migrate:flag-system": "npx tsx scripts/apply-flag-migration.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@libsql/client": "^0.17.3",
|
||||
"@mudbill/duckduckgo-images-api": "^2.0.1",
|
||||
"@tensorflow/tfjs": "^4.22.0",
|
||||
"@tensorflow/tfjs-node": "^4.22.0",
|
||||
"dotenv": "^17.4.2",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"next": "16.2.7",
|
||||
"react": "19.2.4",
|
||||
"react-dom": "19.2.4",
|
||||
"sharp": "^0.34.5",
|
||||
"uuid": "^14.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4",
|
||||
"@testing-library/jest-dom": "^6.9.1",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/jsdom": "^28.0.3",
|
||||
"@types/node": "^20",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"@vitest/coverage-v8": "^4.1.8",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "16.2.7",
|
||||
"jsdom": "^29.1.1",
|
||||
"tailwindcss": "^4",
|
||||
"typescript": "^5",
|
||||
"vitest": "^4.1.8"
|
||||
}
|
||||
}
|
||||
7
postcss.config.mjs
Normal file
7
postcss.config.mjs
Normal file
@@ -0,0 +1,7 @@
|
||||
const config = {
|
||||
plugins: {
|
||||
"@tailwindcss/postcss": {},
|
||||
},
|
||||
};
|
||||
|
||||
export default config;
|
||||
1
public/file.svg
Normal file
1
public/file.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg fill="none" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path d="M14.5 13.5V5.41a1 1 0 0 0-.3-.7L9.8.29A1 1 0 0 0 9.08 0H1.5v13.5A2.5 2.5 0 0 0 4 16h8a2.5 2.5 0 0 0 2.5-2.5m-1.5 0v-7H8v-5H3v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1M9.5 5V2.12L12.38 5zM5.13 5h-.62v1.25h2.12V5zm-.62 3h7.12v1.25H4.5zm.62 3h-.62v1.25h7.12V11z" clip-rule="evenodd" fill="#666" fill-rule="evenodd"/></svg>
|
||||
|
After Width: | Height: | Size: 391 B |
1
public/globe.svg
Normal file
1
public/globe.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g clip-path="url(#a)"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.27 14.1a6.5 6.5 0 0 0 3.67-3.45q-1.24.21-2.7.34-.31 1.83-.97 3.1M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16m.48-1.52a7 7 0 0 1-.96 0H7.5a4 4 0 0 1-.84-1.32q-.38-.89-.63-2.08a40 40 0 0 0 3.92 0q-.25 1.2-.63 2.08a4 4 0 0 1-.84 1.31zm2.94-4.76q1.66-.15 2.95-.43a7 7 0 0 0 0-2.58q-1.3-.27-2.95-.43a18 18 0 0 1 0 3.44m-1.27-3.54a17 17 0 0 1 0 3.64 39 39 0 0 1-4.3 0 17 17 0 0 1 0-3.64 39 39 0 0 1 4.3 0m1.1-1.17q1.45.13 2.69.34a6.5 6.5 0 0 0-3.67-3.44q.65 1.26.98 3.1M8.48 1.5l.01.02q.41.37.84 1.31.38.89.63 2.08a40 40 0 0 0-3.92 0q.25-1.2.63-2.08a4 4 0 0 1 .85-1.32 7 7 0 0 1 .96 0m-2.75.4a6.5 6.5 0 0 0-3.67 3.44 29 29 0 0 1 2.7-.34q.31-1.83.97-3.1M4.58 6.28q-1.66.16-2.95.43a7 7 0 0 0 0 2.58q1.3.27 2.95.43a18 18 0 0 1 0-3.44m.17 4.71q-1.45-.12-2.69-.34a6.5 6.5 0 0 0 3.67 3.44q-.65-1.27-.98-3.1" fill="#666"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h16v16H0z"/></clipPath></defs></svg>
|
||||
|
After Width: | Height: | Size: 1.0 KiB |
0
public/models/.gitkeep
Normal file
0
public/models/.gitkeep
Normal file
1
public/next.svg
Normal file
1
public/next.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
1
public/vercel.svg
Normal file
1
public/vercel.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1155 1000"><path d="m577.3 0 577.4 1000H0z" fill="#fff"/></svg>
|
||||
|
After Width: | Height: | Size: 128 B |
1
public/window.svg
Normal file
1
public/window.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.5 2.5h13v10a1 1 0 0 1-1 1h-11a1 1 0 0 1-1-1zM0 1h16v11.5a2.5 2.5 0 0 1-2.5 2.5h-11A2.5 2.5 0 0 1 0 12.5zm3.75 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5M7 4.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0m1.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5" fill="#666"/></svg>
|
||||
|
After Width: | Height: | Size: 385 B |
53
scripts/apply-flag-migration.ts
Normal file
53
scripts/apply-flag-migration.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* apply-flag-migration.ts
|
||||
*
|
||||
* Applies the flagged_content table migration to Turso.
|
||||
* Run with: npx tsx scripts/apply-flag-migration.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import path from "node:path";
|
||||
|
||||
const envFile =
|
||||
process.env.NODE_ENV === "production" ? "../.env.production" : "../.env.development";
|
||||
dotenv.config({ path: path.resolve(__dirname, envFile) });
|
||||
|
||||
import { createClient } from "@libsql/client";
|
||||
|
||||
async function main() {
|
||||
const db = createClient({
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
});
|
||||
|
||||
console.log("Applying migration: create flagged_content table...");
|
||||
|
||||
await db.execute(`
|
||||
CREATE TABLE IF NOT EXISTS flagged_content (
|
||||
id text PRIMARY KEY NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
content_id text NOT NULL,
|
||||
field_name text NOT NULL,
|
||||
notes text DEFAULT '',
|
||||
flag_count integer DEFAULT 1 NOT NULL,
|
||||
created_at text DEFAULT (datetime('now')) NOT NULL,
|
||||
updated_at text DEFAULT (datetime('now')) NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
await db.execute(`
|
||||
CREATE INDEX IF NOT EXISTS idx_flagged_content_type ON flagged_content (content_type)
|
||||
`);
|
||||
|
||||
await db.execute(`
|
||||
CREATE INDEX IF NOT EXISTS idx_flagged_content_id ON flagged_content (content_id)
|
||||
`);
|
||||
|
||||
console.log("Migration applied successfully.");
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Migration failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
23
scripts/apply-migration.ts
Normal file
23
scripts/apply-migration.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import "dotenv/config";
|
||||
import { createClient } from "@libsql/client";
|
||||
|
||||
async function main() {
|
||||
const db = createClient({
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
});
|
||||
|
||||
console.log("Applying migration: add image_url to diseases...");
|
||||
await db.execute("ALTER TABLE diseases ADD COLUMN image_url TEXT DEFAULT ''");
|
||||
await db.execute("UPDATE diseases SET image_url = '' WHERE image_url IS NULL");
|
||||
|
||||
// Mark migration as applied
|
||||
await db.execute(
|
||||
"INSERT INTO __drizzle_migrations (hash, created_at) VALUES ('0001_add-disease-images', datetime('now'))",
|
||||
);
|
||||
|
||||
console.log("Migration applied successfully.");
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
19
scripts/check-progress.mjs
Normal file
19
scripts/check-progress.mjs
Normal file
@@ -0,0 +1,19 @@
|
||||
import { createClient } from "@libsql/client";
|
||||
const c = createClient({
|
||||
url: process.env.DATABASE_URL,
|
||||
authToken: process.env.DATABASE_TOKEN,
|
||||
});
|
||||
const r = await c.execute("SELECT COUNT(*) as cnt FROM diseases");
|
||||
const r2 = await c.execute(
|
||||
`SELECT SUM(CASE WHEN image_url IS NOT NULL AND image_url != '' THEN 1 ELSE 0 END) as has, SUM(CASE WHEN image_url IS NULL OR image_url = '' THEN 1 ELSE 0 END) as miss FROM diseases`,
|
||||
);
|
||||
const r3 = await c.execute(
|
||||
`SELECT severity, COUNT(*) as total, SUM(CASE WHEN image_url IS NOT NULL AND image_url != '' THEN 1 ELSE 0 END) as has FROM diseases GROUP BY severity ORDER BY severity`,
|
||||
);
|
||||
console.log(
|
||||
`Total: ${r.rows[0].cnt} | With images: ${r2.rows[0].has} | Missing: ${r2.rows[0].miss}`,
|
||||
);
|
||||
for (const row of r3.rows) {
|
||||
console.log(` ${row.severity?.padEnd(10)}: ${row.has}/${row.total}`);
|
||||
}
|
||||
c.close();
|
||||
296
scripts/convert-keras-to-tfjs.py
Normal file
296
scripts/convert-keras-to-tfjs.py
Normal file
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Inspect and convert a .keras plant disease model to TF.js GraphModel format.
|
||||
|
||||
Uses tensorflowjs_converter CLI to avoid Keras version deserialization issues.
|
||||
|
||||
Usage:
|
||||
pip3 install tensorflowjs # also pulls tensorflow as dependency
|
||||
python3 scripts/convert-keras-to-tfjs.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
MODEL_PATH = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
||||
"public",
|
||||
"models",
|
||||
"plant-disease-classifier",
|
||||
"best_mnv2_pv_original.keras",
|
||||
)
|
||||
|
||||
OUTPUT_DIR = os.path.join(
|
||||
os.path.dirname(MODEL_PATH),
|
||||
"tfjs_model",
|
||||
)
|
||||
|
||||
|
||||
def inspect_keras_metadata():
|
||||
"""Read .keras archive metadata without loading the model."""
|
||||
print("=" * 60)
|
||||
print("MODEL INSPECTION (metadata only)")
|
||||
print("=" * 60)
|
||||
|
||||
try:
|
||||
import zipfile
|
||||
except ImportError:
|
||||
print("ERROR: zipfile not available")
|
||||
sys.exit(1)
|
||||
|
||||
if not os.path.exists(MODEL_PATH):
|
||||
print(f"ERROR: Model not found at {MODEL_PATH}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\nModel file: {MODEL_PATH}")
|
||||
print(
|
||||
f"File size: {os.path.getsize(MODEL_PATH):,} bytes ({os.path.getsize(MODEL_PATH) / 1024 / 1024:.1f} MB)"
|
||||
)
|
||||
|
||||
# .keras files are ZIP archives
|
||||
with zipfile.ZipFile(MODEL_PATH) as zf:
|
||||
names = zf.namelist()
|
||||
print(f"\nArchive contents ({len(names)} entries):")
|
||||
for name in names:
|
||||
info = zf.getinfo(name)
|
||||
print(f" {name:<40s} {info.file_size:>10,} bytes")
|
||||
|
||||
# Read config.json for model architecture info
|
||||
config_path = None
|
||||
for name in names:
|
||||
if name.endswith("config.json"):
|
||||
config_path = name
|
||||
break
|
||||
|
||||
if config_path:
|
||||
print(f"\nReading {config_path}...")
|
||||
with zf.open(config_path) as f:
|
||||
config = json.load(f)
|
||||
|
||||
# Extract key info
|
||||
model_type = config.get("class_name", "unknown")
|
||||
print(f"Model class: {model_type}")
|
||||
|
||||
# Try to find output layer info
|
||||
if "config" in config:
|
||||
inner_config = config["config"]
|
||||
|
||||
# Look for output shape in config
|
||||
if "output_shape" in inner_config:
|
||||
print(f"Output shape: {inner_config['output_shape']}")
|
||||
|
||||
# Look through layers for the final dense layer
|
||||
if "layers" in inner_config:
|
||||
layers = inner_config["layers"]
|
||||
print(f"\nLayers ({len(layers)} total):")
|
||||
for layer in layers:
|
||||
layer_name = layer.get("config", {}).get("name", "?")
|
||||
layer_class = layer.get("class_name", "?")
|
||||
layer_module = layer.get("module", "?")
|
||||
|
||||
# Extract units/activation for dense layers
|
||||
layer_config = layer.get("config", {})
|
||||
units = layer_config.get("units")
|
||||
activation = layer_config.get("activation")
|
||||
|
||||
detail = ""
|
||||
if units:
|
||||
detail = f" units={units}"
|
||||
if activation:
|
||||
detail += f" activation={activation}"
|
||||
|
||||
print(f" {layer_name:<30s} {layer_class:<20s}{detail}")
|
||||
|
||||
# Find last dense layer for class count
|
||||
for layer in reversed(layers):
|
||||
if layer.get("class_name") == "Dense":
|
||||
units = layer.get("config", {}).get("units")
|
||||
activation = layer.get("config", {}).get("activation")
|
||||
print("\nClassification head:")
|
||||
print(f" Units (classes): {units}")
|
||||
print(f" Activation: {activation}")
|
||||
print(
|
||||
f" Layer name: {layer.get('config', {}).get('name', '?')}"
|
||||
)
|
||||
break
|
||||
|
||||
# Check compile config
|
||||
if "compile_config" in config:
|
||||
compile_cfg = config["compile_config"]
|
||||
optimizer = compile_cfg.get("optimizer", {})
|
||||
if isinstance(optimizer, dict):
|
||||
opt_name = optimizer.get("class_name", "?")
|
||||
lr = optimizer.get("config", {}).get("learning_rate")
|
||||
print("\nTraining config:")
|
||||
print(f" Optimizer: {opt_name}")
|
||||
if lr:
|
||||
print(f" Learning rate: {lr}")
|
||||
loss = compile_cfg.get("loss", "?")
|
||||
metrics = compile_cfg.get("metrics", [])
|
||||
print(f" Loss: {loss}")
|
||||
print(f" Metrics: {metrics}")
|
||||
|
||||
# Check input shape
|
||||
if "build_config" in config:
|
||||
build_cfg = config["build_config"]
|
||||
if "input_shape" in build_cfg:
|
||||
print(f"\nInput shape: {build_cfg['input_shape']}")
|
||||
|
||||
|
||||
def convert_to_tfjs():
|
||||
"""Convert using tensorflowjs_converter CLI."""
|
||||
print("\n" + "=" * 60)
|
||||
print("CONVERTING TO TF.JS GRAPH MODEL")
|
||||
print("=" * 60)
|
||||
|
||||
# Check tensorflowjs_converter CLI is available
|
||||
converter = shutil.which("tensorflowjs_converter")
|
||||
if not converter:
|
||||
print("ERROR: tensorflowjs_converter not found in PATH.")
|
||||
print(" pip3 install tensorflowjs")
|
||||
sys.exit(1)
|
||||
|
||||
# Clean output dir
|
||||
if os.path.exists(OUTPUT_DIR):
|
||||
print(f"Removing existing output dir: {OUTPUT_DIR}")
|
||||
shutil.rmtree(OUTPUT_DIR)
|
||||
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
|
||||
print(f"\nConverting {MODEL_PATH} -> {OUTPUT_DIR}/")
|
||||
print("(this may take a minute...)")
|
||||
|
||||
# Use the venv's python to run the converter (avoids import issues)
|
||||
python_exe = sys.executable # the python running this script
|
||||
result = subprocess.run(
|
||||
[
|
||||
python_exe,
|
||||
"-m",
|
||||
"tensorflowjs.converters.converter",
|
||||
"--input_format=keras",
|
||||
"--output_format=tfjs_graph_model",
|
||||
MODEL_PATH,
|
||||
OUTPUT_DIR,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
print("\nERROR: Conversion failed!")
|
||||
print(f"stdout: {result.stdout}")
|
||||
print(f"stderr: {result.stderr}")
|
||||
sys.exit(1)
|
||||
|
||||
if result.stdout:
|
||||
print(result.stdout)
|
||||
if result.stderr:
|
||||
# Some warnings are normal
|
||||
print(f"Converter output: {result.stderr}")
|
||||
|
||||
# Verify output
|
||||
model_json_path = os.path.join(OUTPUT_DIR, "model.json")
|
||||
if not os.path.exists(model_json_path):
|
||||
print("ERROR: Conversion did not produce model.json")
|
||||
sys.exit(1)
|
||||
|
||||
# List output files
|
||||
files = os.listdir(OUTPUT_DIR)
|
||||
total_size = sum(
|
||||
os.path.getsize(os.path.join(OUTPUT_DIR, f))
|
||||
for f in files
|
||||
if os.path.isfile(os.path.join(OUTPUT_DIR, f))
|
||||
)
|
||||
|
||||
print("\nConversion complete!")
|
||||
print(f"Output directory: {OUTPUT_DIR}/")
|
||||
print(f"Files: {len(files)}")
|
||||
for f in sorted(files):
|
||||
fpath = os.path.join(OUTPUT_DIR, f)
|
||||
if os.path.isfile(fpath):
|
||||
size = os.path.getsize(fpath)
|
||||
print(f" {f:<30s} {size:>10,} bytes")
|
||||
print(f"Total size: {total_size:,} bytes ({total_size / 1024 / 1024:.1f} MB)")
|
||||
|
||||
# Read model.json to check config
|
||||
with open(model_json_path) as f:
|
||||
model_json = json.load(f)
|
||||
|
||||
print(f"\nTF.js model format: {model_json.get('format', 'unknown')}")
|
||||
print(f"Generated by: {model_json.get('generatedBy', 'unknown')}")
|
||||
|
||||
# Inspect model topology
|
||||
if "modelTopology" in model_json:
|
||||
topology = model_json["modelTopology"]
|
||||
print("\nModel topology:")
|
||||
print(f" Name: {topology.get('model_name', 'unnamed')}")
|
||||
print(f" Ops: {len(topology.get('node', []))} nodes")
|
||||
|
||||
# Input/output nodes
|
||||
inputs = topology.get("inputs", {})
|
||||
outputs = topology.get("outputs", {})
|
||||
print(f" Inputs: {list(inputs.keys())}")
|
||||
for name, info in inputs.items():
|
||||
shape = info.get("tensorShape", {})
|
||||
print(f" {name}: shape={shape.get('dim', 'unknown')}")
|
||||
print(f" Outputs: {list(outputs.keys())}")
|
||||
for name, info in outputs.items():
|
||||
shape = info.get("tensorShape", {})
|
||||
print(f" {name}: shape={shape.get('dim', 'unknown')}")
|
||||
|
||||
# Check weights specification
|
||||
if "weightsManifest" in model_json:
|
||||
manifest = model_json["weightsManifest"]
|
||||
print(f"\nWeight manifests: {len(manifest)}")
|
||||
for i, m in enumerate(manifest):
|
||||
shards = m.get("shards", [])
|
||||
print(f" Manifest {i}: {len(shards)} shard(s)")
|
||||
|
||||
return OUTPUT_DIR
|
||||
|
||||
|
||||
def main():
|
||||
if not os.path.exists(MODEL_PATH):
|
||||
print(f"ERROR: Model not found at {MODEL_PATH}")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 1: Inspect metadata
|
||||
inspect_keras_metadata()
|
||||
|
||||
# Step 2: Convert
|
||||
output_dir = convert_to_tfjs()
|
||||
|
||||
# Step 3: Summary
|
||||
print("\n" + "=" * 60)
|
||||
print("NEXT STEPS")
|
||||
print("=" * 60)
|
||||
print(f"""
|
||||
1. Move the TF.js model to the expected location:
|
||||
The model-loader expects model.json at:
|
||||
public/models/plant-disease-classifier/model.json
|
||||
|
||||
Move files:
|
||||
mv {output_dir}/model.json public/models/plant-disease-classifier/
|
||||
mv {output_dir}/group1-shard* public/models/plant-disease-classifier/
|
||||
|
||||
2. IMPORTANT: This model has 38 output classes (original PlantVillage).
|
||||
Your labels.ts expects 95 classes (93 diseases + healthy + unknown).
|
||||
You'll need to either:
|
||||
a) Fine-tune the model with your 95-class dataset, OR
|
||||
b) Map the 38 PlantVillage classes to your disease IDs
|
||||
|
||||
3. Install @tensorflow/tfjs in your project:
|
||||
npm install @tensorflow/tfjs
|
||||
|
||||
4. Test with your API:
|
||||
npm run dev
|
||||
POST /api/identify with an uploaded image
|
||||
""")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
2337
scripts/disease-templates.ts
Normal file
2337
scripts/disease-templates.ts
Normal file
File diff suppressed because it is too large
Load Diff
691
scripts/expand-diseases.ts
Normal file
691
scripts/expand-diseases.ts
Normal file
@@ -0,0 +1,691 @@
|
||||
/**
|
||||
* Expand DB with comprehensive plant disease list from Wikipedia.
|
||||
*
|
||||
* Reads /tmp/plant_diseases/plant_diseases_comprehensive.txt,
|
||||
* compares against existing DB entries (by name, case-insensitive),
|
||||
* and inserts new entries with reasonable defaults.
|
||||
*
|
||||
* Usage:
|
||||
* cd apps/web && export $(grep -v '^#' .env.development | xargs) && npx tsx scripts/expand-diseases.ts
|
||||
*/
|
||||
|
||||
import "dotenv/config";
|
||||
import { readFileSync } from "fs";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { getDb, closeDb } from "../src/lib/db/index";
|
||||
import { plants, diseases } from "../src/lib/db/schema";
|
||||
import type { CausalAgentType, Severity } from "../src/lib/types";
|
||||
|
||||
// ─── Parse the comprehensive list ─────────────────────────────────────────────
|
||||
|
||||
interface DiseaseEntry {
|
||||
name: string;
|
||||
sourceUrl: string;
|
||||
}
|
||||
|
||||
function parseComprehensiveList(filePath: string): DiseaseEntry[] {
|
||||
const content = readFileSync(filePath, "utf-8");
|
||||
const entries: DiseaseEntry[] = [];
|
||||
const lines = content.split("\n");
|
||||
const nameRe = /^\d+\.\s+(.+)$/;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const nameMatch = lines[i].match(nameRe);
|
||||
if (nameMatch) {
|
||||
const name = nameMatch[1].trim();
|
||||
const urlLine = lines[i + 1]?.trim() || "";
|
||||
// Only add if the next line is a valid URL
|
||||
if (urlLine.startsWith("http")) {
|
||||
entries.push({ name, sourceUrl: urlLine });
|
||||
i++; // skip the URL line
|
||||
} else {
|
||||
entries.push({ name, sourceUrl: "" });
|
||||
}
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
// ─── Infer causal agent type from disease name ────────────────────────────────
|
||||
|
||||
function inferCausalAgent(name: string): CausalAgentType {
|
||||
const lower = name.toLowerCase();
|
||||
|
||||
// Bacterial indicators
|
||||
if (
|
||||
lower.startsWith("bacterial ") ||
|
||||
lower.includes(" xanthomonas") ||
|
||||
lower.includes(" pseudomonas") ||
|
||||
lower.includes(" erwinia") ||
|
||||
lower.includes(" ralstonia") ||
|
||||
lower.includes(" clavibacter") ||
|
||||
lower.includes(" streptomyces") ||
|
||||
lower.includes(" agrobacterium") ||
|
||||
lower.includes(" corynebacterium") ||
|
||||
lower.includes(" pectobacterium") ||
|
||||
lower.includes(" dickeya")
|
||||
) {
|
||||
return "bacterial";
|
||||
}
|
||||
|
||||
// Viral indicators - strong signals
|
||||
if (
|
||||
lower.includes(" mosaic") ||
|
||||
lower.includes(" yellows") ||
|
||||
lower.includes(" leaf roll") ||
|
||||
lower.includes(" leafroll") ||
|
||||
lower.includes(" ringspot") ||
|
||||
lower.includes(" ring spot") ||
|
||||
lower.includes(" enation") ||
|
||||
lower.includes(" phyllody") ||
|
||||
lower.includes(" witches") ||
|
||||
lower.includes(" witches'") ||
|
||||
lower.includes(" crinkle") ||
|
||||
lower.includes(" rosette") ||
|
||||
lower.includes(" shoestring") ||
|
||||
lower.includes(" tristeza") ||
|
||||
lower.includes(" psorosis") ||
|
||||
lower.includes(" stubborn") ||
|
||||
lower.includes(" greening") ||
|
||||
lower.includes(" vein banding") ||
|
||||
lower.includes(" vein mottle") ||
|
||||
lower.includes(" vein clearing") ||
|
||||
lower.includes(" leaf pucker") ||
|
||||
lower.includes(" pucker leaf") ||
|
||||
lower.includes(" latent") ||
|
||||
lower.includes(" motley") ||
|
||||
lower.includes(" rugose")
|
||||
) {
|
||||
return "viral";
|
||||
}
|
||||
|
||||
// Viral - names containing "virus" or "viroid"
|
||||
if (lower.includes(" virus") || lower.includes(" viroid") || lower.includes(" virosis")) {
|
||||
return "viral";
|
||||
}
|
||||
|
||||
// Nematodes
|
||||
if (
|
||||
lower.includes(" nematode") ||
|
||||
lower.includes(" nematodes") ||
|
||||
lower.includes(" eelworm") ||
|
||||
lower.includes(" root knot") ||
|
||||
lower.includes(" root-knot") ||
|
||||
lower.includes(" cyst ") ||
|
||||
lower.includes(" dagger ") ||
|
||||
lower.includes(" lance ") ||
|
||||
lower.includes(" lesion ") ||
|
||||
lower.includes(" ring ") ||
|
||||
lower.includes(" spiral ") ||
|
||||
lower.includes(" sting ") ||
|
||||
lower.includes(" stubby ") ||
|
||||
lower.includes(" needle ") ||
|
||||
lower.includes(" foliar ") ||
|
||||
lower.includes(" bulb ") ||
|
||||
lower.includes(" reniform ") ||
|
||||
lower.includes(" burrowing ")
|
||||
) {
|
||||
// Check if it's really a nematode name
|
||||
if (lower.includes("nematode")) return "environmental";
|
||||
}
|
||||
|
||||
// Fungal indicators
|
||||
if (
|
||||
lower.includes(" mildew") ||
|
||||
lower.includes(" rust") ||
|
||||
lower.includes(" smut") ||
|
||||
lower.includes(" blight") ||
|
||||
lower.includes(" canker") ||
|
||||
lower.includes(" rot") ||
|
||||
lower.includes(" scab") ||
|
||||
lower.includes(" mold") ||
|
||||
lower.includes(" anthracnose") ||
|
||||
lower.includes(" bunt") ||
|
||||
lower.includes(" ergot") ||
|
||||
lower.includes(" dieback") ||
|
||||
lower.includes(" scald") ||
|
||||
lower.includes(" blotch") ||
|
||||
lower.includes(" speckle") ||
|
||||
lower.includes(" sooty") ||
|
||||
lower.includes(" flyspeck") ||
|
||||
lower.includes(" fusarium") ||
|
||||
lower.includes(" alternaria") ||
|
||||
lower.includes(" botrytis") ||
|
||||
lower.includes(" rhizoctonia") ||
|
||||
lower.includes(" pythium") ||
|
||||
lower.includes(" phytophthora") ||
|
||||
lower.includes(" sclerotinia") ||
|
||||
lower.includes(" verticillium") ||
|
||||
lower.includes(" ascochyta") ||
|
||||
lower.includes(" cercospora") ||
|
||||
lower.includes(" septoria") ||
|
||||
lower.includes(" colletotrichum") ||
|
||||
lower.includes(" phomopsis") ||
|
||||
lower.includes(" diaporthe") ||
|
||||
lower.includes(" diplodia") ||
|
||||
lower.includes(" macrophomina") ||
|
||||
lower.includes(" cylindrocladium") ||
|
||||
lower.includes(" mycosphaerella") ||
|
||||
lower.includes(" helminthosporium") ||
|
||||
lower.includes(" curvularia") ||
|
||||
lower.includes(" bipolaris") ||
|
||||
lower.includes(" exserohilum") ||
|
||||
lower.includes(" dothiorella") ||
|
||||
lower.includes(" fusicoccum") ||
|
||||
lower.includes(" pestalotia") ||
|
||||
lower.includes(" glomerella") ||
|
||||
lower.includes(" nectria") ||
|
||||
lower.includes(" eutypa") ||
|
||||
lower.includes(" armillaria") ||
|
||||
lower.includes(" ganoderma") ||
|
||||
lower.includes(" phoma") ||
|
||||
lower.includes(" cladosporium") ||
|
||||
lower.includes(" penicillium") ||
|
||||
lower.includes(" aspergillus") ||
|
||||
lower.includes(" rhizopus") ||
|
||||
lower.includes(" mucor") ||
|
||||
lower.includes(" downy mildew") ||
|
||||
lower.includes(" powdery mildew") ||
|
||||
lower.includes(" pink rot") ||
|
||||
lower.includes(" pink mold") ||
|
||||
lower.includes(" pink root") ||
|
||||
lower.includes(" gray mold") ||
|
||||
lower.includes(" grey mold") ||
|
||||
lower.includes(" white rot") ||
|
||||
lower.includes(" white mold") ||
|
||||
lower.includes(" brown rot") ||
|
||||
lower.includes(" black rot") ||
|
||||
lower.includes(" soft rot") ||
|
||||
lower.includes(" dry rot") ||
|
||||
lower.includes(" fruit rot") ||
|
||||
lower.includes(" root rot") ||
|
||||
lower.includes(" stem rot") ||
|
||||
lower.includes(" ear rot") ||
|
||||
lower.includes(" crown rot") ||
|
||||
lower.includes(" collar rot") ||
|
||||
lower.includes(" pod rot") ||
|
||||
lower.includes(" kernel rot") ||
|
||||
lower.includes(" stalk rot") ||
|
||||
lower.includes(" head rot") ||
|
||||
lower.includes(" butt rot") ||
|
||||
lower.includes(" stump rot") ||
|
||||
lower.includes(" wood rot") ||
|
||||
lower.includes(" seed rot") ||
|
||||
lower.includes(" leaf spot") ||
|
||||
lower.includes(" leaf blight") ||
|
||||
lower.includes(" leaf blotch") ||
|
||||
lower.includes(" leaf rust") ||
|
||||
lower.includes(" brown spot") ||
|
||||
lower.includes(" black spot") ||
|
||||
lower.includes(" black leg") ||
|
||||
lower.includes(" blackleg") ||
|
||||
lower.includes(" black foot") ||
|
||||
lower.includes(" white rust") ||
|
||||
lower.includes(" white smut") ||
|
||||
lower.includes(" white scab") ||
|
||||
lower.includes(" tar spot") ||
|
||||
lower.includes(" target spot") ||
|
||||
lower.includes(" dollar spot") ||
|
||||
lower.includes(" fairy ring") ||
|
||||
lower.includes(" snow mold") ||
|
||||
lower.includes(" pink disease") ||
|
||||
lower.includes(" thread blight") ||
|
||||
lower.includes(" web blight") ||
|
||||
lower.includes(" sclerotial") ||
|
||||
lower.includes(" sore shin") ||
|
||||
lower.includes(" wart") ||
|
||||
lower.includes(" scurf") ||
|
||||
lower.includes(" silver scurf") ||
|
||||
lower.includes(" shot hole") ||
|
||||
lower.includes(" timber rot") ||
|
||||
lower.includes(" cottony rot") ||
|
||||
lower.includes(" watery rot") ||
|
||||
lower.includes(" sour rot") ||
|
||||
lower.includes(" seepage") ||
|
||||
lower.includes(" bunch rot") ||
|
||||
lower.includes(" noble rot") ||
|
||||
lower.includes(" bitter rot") ||
|
||||
lower.includes(" ripe rot") ||
|
||||
lower.includes(" ring rot") ||
|
||||
lower.includes(" coral spot") ||
|
||||
lower.includes(" stem canker") ||
|
||||
lower.includes(" branch canker") ||
|
||||
lower.includes(" perennial canker") ||
|
||||
lower.includes(" brand canker") ||
|
||||
lower.includes(" blister canker") ||
|
||||
lower.includes(" bleeding canker") ||
|
||||
lower.includes(" bark canker") ||
|
||||
lower.includes(" gum canker") ||
|
||||
lower.includes(" collar crack") ||
|
||||
lower.includes(" fasciation") ||
|
||||
lower.includes(" exobasidium") ||
|
||||
lower.includes(" mycorrhiza") ||
|
||||
lower.includes(" lichen") ||
|
||||
lower.includes(" algal") ||
|
||||
lower.includes(" chlorosis") ||
|
||||
lower.includes(" leaf blister") ||
|
||||
lower.includes(" leaf curl")
|
||||
) {
|
||||
return "fungal";
|
||||
}
|
||||
|
||||
// Physiological / environmental indicators
|
||||
if (
|
||||
lower.includes(" sunscald") ||
|
||||
lower.includes(" sunburn") ||
|
||||
lower.includes(" chilling") ||
|
||||
lower.includes(" blossom end rot") ||
|
||||
lower.includes(" edema") ||
|
||||
lower.includes(" deficiency") ||
|
||||
lower.includes(" toxicity") ||
|
||||
lower.includes(" ozone") ||
|
||||
lower.includes(" drought") ||
|
||||
lower.includes(" frost") ||
|
||||
lower.includes(" herbicide") ||
|
||||
lower.includes(" pesticide") ||
|
||||
lower.includes(" phytotoxicity") ||
|
||||
lower.includes(" catface") ||
|
||||
lower.includes(" fruit cracking") ||
|
||||
lower.includes(" russeting") ||
|
||||
lower.includes(" growth crack") ||
|
||||
lower.includes(" mealiness") ||
|
||||
lower.includes(" wind scar") ||
|
||||
lower.includes(" hail") ||
|
||||
lower.includes(" salt ") ||
|
||||
lower.includes(" nutritional") ||
|
||||
lower.includes(" mineral") ||
|
||||
lower.includes(" overwatering") ||
|
||||
lower.includes(" under watering") ||
|
||||
lower.includes(" waterlogging") ||
|
||||
lower.includes(" chemical injury") ||
|
||||
lower.includes(" spray injury") ||
|
||||
lower.includes(" fertilizer burn") ||
|
||||
lower.includes(" lightning") ||
|
||||
lower.includes(" bruising") ||
|
||||
lower.includes(" pressure bruise") ||
|
||||
lower.includes(" impact damage") ||
|
||||
lower.includes(" transit rot")
|
||||
) {
|
||||
return "environmental";
|
||||
}
|
||||
|
||||
// Insect/mite/pest indicators
|
||||
if (
|
||||
lower.includes(" mite") ||
|
||||
lower.includes(" beetle") ||
|
||||
lower.includes(" weevil") ||
|
||||
lower.includes(" aphid") ||
|
||||
lower.includes(" bollworm") ||
|
||||
lower.includes(" leaf miner") ||
|
||||
lower.includes(" mealybug") ||
|
||||
lower.includes(" thrips") ||
|
||||
lower.includes(" whitefly") ||
|
||||
lower.includes(" caterpillar") ||
|
||||
lower.includes(" sawfly") ||
|
||||
lower.includes(" scale ") ||
|
||||
lower.includes(" leafhopper") ||
|
||||
lower.includes(" psylla") ||
|
||||
lower.includes(" slug") ||
|
||||
lower.includes(" snail") ||
|
||||
lower.includes(" borer") ||
|
||||
lower.includes(" maggot") ||
|
||||
lower.includes(" grub") ||
|
||||
lower.includes(" earwig") ||
|
||||
lower.includes(" grasshopper")
|
||||
) {
|
||||
return "environmental";
|
||||
}
|
||||
|
||||
// Fungal genus names
|
||||
const fungalGenera = [
|
||||
"armillaria",
|
||||
"aspergillus",
|
||||
"alternaria",
|
||||
"botrytis",
|
||||
"cercospora",
|
||||
"cladosporium",
|
||||
"colletotrichum",
|
||||
"curvularia",
|
||||
"cylindrocladium",
|
||||
"diplodia",
|
||||
"fusarium",
|
||||
"ganoderma",
|
||||
"glomerella",
|
||||
"helminthosporium",
|
||||
"macrophomina",
|
||||
"mycosphaerella",
|
||||
"nectria",
|
||||
"penicillium",
|
||||
"pestalotia",
|
||||
"phoma",
|
||||
"phomopsis",
|
||||
"phytophthora",
|
||||
"pythium",
|
||||
"rhizoctonia",
|
||||
"sclerotinia",
|
||||
"septoria",
|
||||
"verticillium",
|
||||
"ascochyta",
|
||||
"cercoseptoria",
|
||||
"phaeoisariopsis",
|
||||
"phaeoseptoria",
|
||||
"stagonospora",
|
||||
"stemphylium",
|
||||
"myrothecium",
|
||||
"myriogenospora",
|
||||
"dactuliophora",
|
||||
"dilophospora",
|
||||
"coniothecium",
|
||||
"coniosporium",
|
||||
"cryptostictis",
|
||||
"catacauma",
|
||||
"botryodiplodia",
|
||||
"botryosphaeria",
|
||||
"cephalosporium",
|
||||
"ceratocystis",
|
||||
"chalara",
|
||||
"choanephora",
|
||||
"clitocybe",
|
||||
"coprinus",
|
||||
"cordana",
|
||||
"corticium",
|
||||
"corynespora",
|
||||
"coryneum",
|
||||
"cylindrocarpon",
|
||||
"cylindrocladiella",
|
||||
"cylindrosporium",
|
||||
"cytospora",
|
||||
"cytosporina",
|
||||
"dematophora",
|
||||
"didymella",
|
||||
"dothiorella",
|
||||
"drechslera",
|
||||
"endothia",
|
||||
"eutypa",
|
||||
"eutypella",
|
||||
"exobasidium",
|
||||
"fusicladium",
|
||||
"fusicoccum",
|
||||
"gibberella",
|
||||
"glomerella",
|
||||
"gnomonia",
|
||||
"graphiola",
|
||||
"guignardia",
|
||||
"hendersonia",
|
||||
"hendersonula",
|
||||
"hymenochaete",
|
||||
"hypoxylon",
|
||||
"lasiodiplodia",
|
||||
"leptosphaeria",
|
||||
"leucostoma",
|
||||
"lophodermium",
|
||||
"macrophoma",
|
||||
"marasmiellus",
|
||||
"marasmius",
|
||||
"massaria",
|
||||
"monilia",
|
||||
"monosporascus",
|
||||
"mystrosporium",
|
||||
"neocosmospora",
|
||||
"nigrospora",
|
||||
"omphalia",
|
||||
"ophiobolus",
|
||||
"ovulinia",
|
||||
"ozonium",
|
||||
"panagrolaimus",
|
||||
"periconia",
|
||||
"pestalosphaeria",
|
||||
"pestalotiopsis",
|
||||
"phialophora",
|
||||
"phymatotrichum",
|
||||
"physalospora",
|
||||
"phytophthora",
|
||||
"plasmodiophora",
|
||||
"plectosporium",
|
||||
"polyporus",
|
||||
"poria",
|
||||
"pseudocercosporella",
|
||||
"pseudopeziza",
|
||||
"pseudoseptoria",
|
||||
"puccinia",
|
||||
"pyrenochaeta",
|
||||
"pythium",
|
||||
"ramularia",
|
||||
"rhizoctonia",
|
||||
"rhizopus",
|
||||
"rhynchosporium",
|
||||
"rosellinia",
|
||||
"sclerophthora",
|
||||
"sclerotinia",
|
||||
"sclerotium",
|
||||
"septoria",
|
||||
"sphaceloma",
|
||||
"sphaeropsis",
|
||||
"spongospora",
|
||||
"stagonospora",
|
||||
"stemphylium",
|
||||
"stereum",
|
||||
"stigmina",
|
||||
"thanatephorus",
|
||||
"thielaviopsis",
|
||||
"tippula",
|
||||
"typhula",
|
||||
"ulocladium",
|
||||
"uredo",
|
||||
"ustilago",
|
||||
"valsa",
|
||||
"venturia",
|
||||
"verticillium",
|
||||
"xylaria",
|
||||
];
|
||||
for (const genus of fungalGenera) {
|
||||
if (lower.includes(genus)) return "fungal";
|
||||
}
|
||||
|
||||
// Default to fungal (most plant diseases are fungal)
|
||||
return "fungal";
|
||||
}
|
||||
|
||||
// ─── Infer severity ───────────────────────────────────────────────────────────
|
||||
|
||||
function inferSeverity(name: string): Severity {
|
||||
const lower = name.toLowerCase();
|
||||
if (
|
||||
lower.includes(" lethal") ||
|
||||
lower.includes(" devastating") ||
|
||||
lower.includes(" destructive") ||
|
||||
lower.includes(" fatal") ||
|
||||
lower.includes(" severe") ||
|
||||
lower.includes(" blight") ||
|
||||
lower.includes(" wilt") ||
|
||||
lower.includes(" canker") ||
|
||||
lower.includes(" dieback") ||
|
||||
lower.includes(" decline") ||
|
||||
lower.includes(" rot") ||
|
||||
lower.includes(" gall") ||
|
||||
lower.includes(" gummosis") ||
|
||||
lower.includes(" necrosis") ||
|
||||
lower.includes(" erwinia")
|
||||
) {
|
||||
return "high";
|
||||
}
|
||||
if (
|
||||
lower.includes(" minor") ||
|
||||
lower.includes(" mild") ||
|
||||
lower.includes(" slight") ||
|
||||
lower.includes(" speckle") ||
|
||||
lower.includes(" fleck") ||
|
||||
lower.includes(" freckle") ||
|
||||
lower.includes(" chlorosis") ||
|
||||
lower.includes(" translucence") ||
|
||||
lower.includes(" superficial")
|
||||
) {
|
||||
return "low";
|
||||
}
|
||||
return "moderate";
|
||||
}
|
||||
|
||||
// ─── Generate a deterministic slug ────────────────────────────────────────────
|
||||
|
||||
function toSlug(name: string): string {
|
||||
return (
|
||||
"wiki-" +
|
||||
name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "")
|
||||
.replace(/-+/g, "-")
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const db = getDb();
|
||||
|
||||
// 1. Get existing disease names from DB
|
||||
const existingDiseases = await db.select({ name: diseases.name }).from(diseases);
|
||||
const existingNames = new Set(existingDiseases.map((d) => d.name.toLowerCase().trim()));
|
||||
|
||||
console.log(`Existing diseases in DB: ${existingNames.size}`);
|
||||
|
||||
// 2. Parse the comprehensive list
|
||||
const entries = parseComprehensiveList("/tmp/plant_diseases/plant_diseases_comprehensive.txt");
|
||||
console.log(`Total entries in comprehensive file: ${entries.length}`);
|
||||
|
||||
// 3. Find or create catch-all plants
|
||||
for (const plantId of ["general", "unknown"]) {
|
||||
const existing = await db.select().from(plants).where(eq(plants.id, plantId)).get();
|
||||
|
||||
if (!existing) {
|
||||
console.log(`Creating '${plantId}' plant for catch-all diseases...`);
|
||||
await db.insert(plants).values({
|
||||
id: plantId,
|
||||
commonName: plantId === "general" ? "General (Multiple Plants)" : "Unknown Plant",
|
||||
scientificName: "Various",
|
||||
family: "Various",
|
||||
category: "houseplant",
|
||||
careSummary:
|
||||
plantId === "general"
|
||||
? "General plant diseases affecting multiple species."
|
||||
: "Plant disease with unknown host plant.",
|
||||
imageUrl: "",
|
||||
});
|
||||
console.log(`Created '${plantId}' plant.`);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Filter new entries (deduplicate within file + against DB)
|
||||
const newEntries: DiseaseEntry[] = [];
|
||||
const skipped: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const entry of entries) {
|
||||
const key = entry.name.toLowerCase().trim();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
if (existingNames.has(key)) {
|
||||
skipped.push(entry.name);
|
||||
} else {
|
||||
newEntries.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nNew entries to insert: ${newEntries.length}`);
|
||||
console.log(`Already existing (skipped): ${skipped.length}`);
|
||||
|
||||
if (skipped.length > 0) {
|
||||
console.log(`\nFirst 10 skipped (of ${skipped.length}):`);
|
||||
skipped.slice(0, 10).forEach((s) => console.log(` - ${s}`));
|
||||
}
|
||||
|
||||
// 5. Insert new entries in batches
|
||||
if (newEntries.length === 0) {
|
||||
console.log("\n✅ No new diseases to insert.");
|
||||
closeDb();
|
||||
return;
|
||||
}
|
||||
|
||||
const BATCH_SIZE = 50;
|
||||
let inserted = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (let i = 0; i < newEntries.length; i += BATCH_SIZE) {
|
||||
const batch = newEntries.slice(i, i + BATCH_SIZE);
|
||||
const values = batch.map((entry) => {
|
||||
const causalAgent = inferCausalAgent(entry.name);
|
||||
const severity = inferSeverity(entry.name);
|
||||
return {
|
||||
id: toSlug(entry.name),
|
||||
plantId: "general",
|
||||
name: entry.name,
|
||||
scientificName: "",
|
||||
causalAgentType: causalAgent,
|
||||
description: `A plant disease known as "${entry.name}". Source: Wikipedia.`,
|
||||
symptoms: [],
|
||||
causes: [],
|
||||
treatment: [],
|
||||
prevention: [],
|
||||
lookalikeIds: [],
|
||||
severity,
|
||||
sourceUrl: entry.sourceUrl,
|
||||
imageUrl: "",
|
||||
};
|
||||
});
|
||||
|
||||
try {
|
||||
await db.insert(diseases).values(values).onConflictDoNothing();
|
||||
inserted += values.length;
|
||||
} catch (err) {
|
||||
// Fall back to individual inserts for this batch if batch fails
|
||||
console.log(` Batch failed, trying individually...`);
|
||||
for (const val of values) {
|
||||
try {
|
||||
await db.insert(diseases).values(val).onConflictDoNothing();
|
||||
inserted++;
|
||||
} catch (e2) {
|
||||
// If it's a duplicate key, count it as skipped
|
||||
if (String(e2).includes("UNIQUE") || String(e2).includes("duplicate")) {
|
||||
// Already handled by onConflictDoNothing, shouldn't happen
|
||||
inserted++;
|
||||
} else {
|
||||
console.error(` Error inserting "${val.name}":`, e2);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((i + BATCH_SIZE) % 200 === 0 || i + BATCH_SIZE >= newEntries.length) {
|
||||
console.log(
|
||||
` Progress: ${Math.min(i + BATCH_SIZE, newEntries.length)}/${newEntries.length} (${inserted} inserted, ${errors} errors)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Summary
|
||||
const totalDiseases = await db
|
||||
.select({ count: sql<number>`COUNT(*)` })
|
||||
.from(diseases)
|
||||
.get();
|
||||
const totalPlants = await db
|
||||
.select({ count: sql<number>`COUNT(*)` })
|
||||
.from(plants)
|
||||
.get();
|
||||
|
||||
console.log(`\n📊 Results:`);
|
||||
console.log(` Inserted: ${inserted}`);
|
||||
console.log(` Errors: ${errors}`);
|
||||
console.log(` Skipped (already existed): ${skipped.length}`);
|
||||
console.log(`\n📊 Database now has:`);
|
||||
console.log(` ${totalPlants?.count ?? 0} plants`);
|
||||
console.log(` ${totalDiseases?.count ?? 0} diseases`);
|
||||
|
||||
closeDb();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
414
scripts/fill-brave-images-v2.ts
Normal file
414
scripts/fill-brave-images-v2.ts
Normal file
@@ -0,0 +1,414 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* fill-brave-images-v2.ts — Brave Image Search for remaining disease images.
|
||||
*
|
||||
* Prioritizes by severity (critical → high → moderate → low).
|
||||
* Runs at 1 request/sec (Brave free tier rate limit).
|
||||
* Updates Turso DB directly with found images.
|
||||
* When current key is exhausted, prompts for next key.
|
||||
* Falls back to duckduckgo-images-api when all keys are spent.
|
||||
*
|
||||
* Usage:
|
||||
* cd apps/web && npx tsx scripts/fill-brave-images-v2.ts
|
||||
*
|
||||
* Pass additional API keys as args:
|
||||
* npx tsx scripts/fill-brave-images-v2.ts KEY2 KEY3
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
|
||||
// Load env
|
||||
const envPath = resolve(__dirname, "../.env.development");
|
||||
try {
|
||||
const env = readFileSync(envPath, "utf-8");
|
||||
for (const line of env.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed && !trimmed.startsWith("#")) {
|
||||
const eqIdx = trimmed.indexOf("=");
|
||||
if (eqIdx > 0) {
|
||||
const key = trimmed.slice(0, eqIdx).trim();
|
||||
const val = trimmed.slice(eqIdx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Also try .env.local for BRAVE_API_KEY
|
||||
try {
|
||||
const envLocal = readFileSync(resolve(__dirname, "../.env.local"), "utf-8");
|
||||
for (const line of envLocal.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith("BRAVE_API_KEY=")) {
|
||||
const val = trimmed.slice("BRAVE_API_KEY=".length).trim();
|
||||
if (!process.env.BRAVE_API_KEY) process.env.BRAVE_API_KEY = val;
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
import { getDb, closeDb } from "../src/lib/db/index";
|
||||
import { diseases } from "../src/lib/db/schema";
|
||||
import { createClient } from "@libsql/client";
|
||||
import { sql } from "drizzle-orm";
|
||||
|
||||
interface DiseaseRow {
|
||||
id: string;
|
||||
name: string;
|
||||
scientificName: string;
|
||||
severity: string;
|
||||
plantId: string;
|
||||
}
|
||||
|
||||
// ─── Config ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const BRAVE_DELAY = 1100; // ms between calls (1 req/sec)
|
||||
const DB_FLUSH_BATCH = 50;
|
||||
const MAX_PER_KEY = 1800; // Leave 200 buffer of the 2000/mo limit
|
||||
const STATE_FILE = resolve(__dirname, ".brave-progress.json");
|
||||
|
||||
let currentKeyIndex = 0;
|
||||
let braveKeys: string[] = [];
|
||||
let callsThisKey = 0;
|
||||
let totalFound = 0;
|
||||
// totalSkipped tracking removed — not needed for v2
|
||||
|
||||
// ─── State persistence ───────────────────────────────────────────────────────
|
||||
|
||||
interface RunState {
|
||||
processedIds: string[];
|
||||
currentKeyIndex: number;
|
||||
callsThisKey: number;
|
||||
totalFound: number;
|
||||
}
|
||||
|
||||
function loadState(): RunState | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(STATE_FILE, "utf-8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function saveState(processedIds: string[]) {
|
||||
writeFileSync(
|
||||
STATE_FILE,
|
||||
JSON.stringify(
|
||||
{
|
||||
processedIds,
|
||||
currentKeyIndex,
|
||||
callsThisKey,
|
||||
totalFound,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf-8",
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Brave API ───────────────────────────────────────────────────────────────
|
||||
|
||||
async function braveImageSearch(query: string): Promise<string | null> {
|
||||
const key = braveKeys[currentKeyIndex];
|
||||
if (!key) return null;
|
||||
|
||||
const url = new URL("https://api.search.brave.com/res/v1/images/search");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("count", "3");
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
const res = await fetch(url.toString(), {
|
||||
headers: { "X-Subscription-Token": key, Accept: "application/json" },
|
||||
});
|
||||
|
||||
if (res.status === 429) {
|
||||
console.log("\n [RATE LIMITED] Key " + (currentKeyIndex + 1) + " exhausted!");
|
||||
return "RATE_LIMITED";
|
||||
}
|
||||
if (!res.ok) return null;
|
||||
|
||||
callsThisKey++;
|
||||
const data = (await res.json()) as {
|
||||
results?: Array<{ url: string; thumbnail?: { src?: string } }>;
|
||||
};
|
||||
const results = data?.results ?? [];
|
||||
if (results.length === 0) return null;
|
||||
|
||||
// Prefer non-stock images
|
||||
for (const r of results) {
|
||||
const src = r.thumbnail?.src ?? r.url;
|
||||
if (src && !/(dreamstime|shutterstock|alamy|istock|123rf)/i.test(src)) {
|
||||
return src;
|
||||
}
|
||||
}
|
||||
return results[0].thumbnail?.src ?? results[0].url;
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── DuckDuckGo fallback ────────────────────────────────────────────────────
|
||||
|
||||
async function ddgFallbackSearch(query: string): Promise<string | null> {
|
||||
try {
|
||||
// Try to use duckduckgo-images-api if installed
|
||||
const ddg = await import("duckduckgo-images-api").catch(() => null);
|
||||
if (ddg) {
|
||||
const results = await ddg.image_search({ query, moderate: true });
|
||||
if (results && results.length > 0) {
|
||||
for (const r of results) {
|
||||
if (r.image && !/(dreamstime|shutterstock|alamy|istock|123rf)/i.test(r.image)) {
|
||||
return r.image;
|
||||
}
|
||||
}
|
||||
return results[0].image || null;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// duckduckgo-images-api not installed
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
console.log("\n🔍 Brave Disease Image Filler v2\n");
|
||||
|
||||
// Parse keys from args + env
|
||||
const argsKeys = process.argv.slice(2).filter((a) => !a.startsWith("-"));
|
||||
const envKey = process.env.BRAVE_API_KEY;
|
||||
braveKeys = [envKey, ...argsKeys].filter(Boolean) as string[];
|
||||
braveKeys = [...new Set(braveKeys)]; // dedup
|
||||
|
||||
if (braveKeys.length === 0) {
|
||||
console.log("❌ No Brave API keys found.");
|
||||
console.log(" Set BRAVE_API_KEY in .env.local or pass as argument.\n");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`🔑 ${braveKeys.length} Brave API key(s) available\n`);
|
||||
|
||||
// Load state
|
||||
const state = loadState();
|
||||
if (state) {
|
||||
currentKeyIndex = state.currentKeyIndex;
|
||||
callsThisKey = state.callsThisKey;
|
||||
totalFound = state.totalFound;
|
||||
console.log(
|
||||
`📋 Resuming from previous run (${state.processedIds.length} processed, ${totalFound} found)\n`,
|
||||
);
|
||||
}
|
||||
|
||||
// Get diseases from DB
|
||||
const db = getDb();
|
||||
const allDiseases = (await db
|
||||
.select({
|
||||
id: diseases.id,
|
||||
name: diseases.name,
|
||||
scientificName: diseases.scientificName,
|
||||
severity: diseases.severity,
|
||||
plantId: diseases.plantId,
|
||||
})
|
||||
.from(diseases)
|
||||
.where(sql`(image_url IS NULL OR image_url = '')`)
|
||||
.all()) as DiseaseRow[];
|
||||
|
||||
console.log(`📋 ${allDiseases.length} diseases need images\n`);
|
||||
|
||||
if (allDiseases.length === 0) {
|
||||
console.log("✅ All diseases already have images!\n");
|
||||
closeDb();
|
||||
return;
|
||||
}
|
||||
|
||||
// Sort by severity priority
|
||||
const severityOrder = { critical: 0, high: 1, moderate: 2, low: 3 };
|
||||
allDiseases.sort(
|
||||
(a, b) =>
|
||||
(severityOrder[a.severity as keyof typeof severityOrder] || 99) -
|
||||
(severityOrder[b.severity as keyof typeof severityOrder] || 99),
|
||||
);
|
||||
|
||||
// Filter out already-processed from state
|
||||
const processedSet = new Set(state?.processedIds || []);
|
||||
const pending = allDiseases.filter((d) => !processedSet.has(d.id));
|
||||
|
||||
console.log(
|
||||
`📊 Prioritization: critical=${allDiseases.filter((d) => d.severity === "critical" && !processedSet.has(d.id)).length}, high=${allDiseases.filter((d) => d.severity === "high" && !processedSet.has(d.id)).length}, moderate=${allDiseases.filter((d) => d.severity === "moderate" && !processedSet.has(d.id)).length}, low=${allDiseases.filter((d) => d.severity === "low" && !processedSet.has(d.id)).length}\n`,
|
||||
);
|
||||
|
||||
if (pending.length === 0) {
|
||||
console.log("✅ All remaining diseases already attempted\n");
|
||||
closeDb();
|
||||
return;
|
||||
}
|
||||
|
||||
const raw = createClient({
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
});
|
||||
|
||||
let updates: Array<{ id: string; url: string }> = [];
|
||||
const processedIds: string[] = state?.processedIds || [];
|
||||
let found = totalFound;
|
||||
let ddgMode = false;
|
||||
|
||||
for (let i = 0; i < pending.length; i++) {
|
||||
const d = pending[i];
|
||||
|
||||
// Check if current key needs rotating
|
||||
if (!ddgMode && callsThisKey >= MAX_PER_KEY) {
|
||||
if (currentKeyIndex < braveKeys.length - 1) {
|
||||
currentKeyIndex++;
|
||||
callsThisKey = 0;
|
||||
console.log(`\n 🔄 Rotating to key ${currentKeyIndex + 1}/${braveKeys.length}\n`);
|
||||
} else {
|
||||
console.log(
|
||||
`\n ⚠️ All ${braveKeys.length} Brave keys exhausted. Switching to DuckDuckGo fallback.\n`,
|
||||
);
|
||||
ddgMode = true;
|
||||
// Install duckduckgo-images-api if not available
|
||||
try {
|
||||
await import("duckduckgo-images-api");
|
||||
} catch {
|
||||
console.log(" Installing duckduckgo-images-api...");
|
||||
const { execSync } = await import("child_process");
|
||||
execSync("npm install duckduckgo-images-api", {
|
||||
cwd: resolve(__dirname, ".."),
|
||||
stdio: "pipe",
|
||||
});
|
||||
console.log(" Done.\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build search query
|
||||
const plantName = d.plantId.replace(/-/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
const query = `${d.name} ${d.scientificName} ${plantName} plant disease`;
|
||||
const sev = d.severity.padEnd(8);
|
||||
|
||||
process.stdout.write(
|
||||
` [${String(i + 1).padStart(4)}/${pending.length}] [${sev}] ${d.name.substring(0, 40).padEnd(42)} `,
|
||||
);
|
||||
|
||||
let url: string | null = null;
|
||||
|
||||
if (ddgMode) {
|
||||
url = await ddgFallbackSearch(query);
|
||||
if (!url) {
|
||||
// Try a simpler query
|
||||
url = await ddgFallbackSearch(`${d.name} disease`);
|
||||
}
|
||||
} else {
|
||||
url = await braveImageSearch(query);
|
||||
if (url === "RATE_LIMITED") {
|
||||
// Key exhausted mid-query, try next
|
||||
if (currentKeyIndex < braveKeys.length - 1) {
|
||||
currentKeyIndex++;
|
||||
callsThisKey = 0;
|
||||
console.log("\n 🔄 Rotating key...");
|
||||
url = await braveImageSearch(query);
|
||||
} else {
|
||||
console.log("\n ⚠️ All keys exhausted mid-batch!");
|
||||
ddgMode = true;
|
||||
url = await ddgFallbackSearch(query);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (url) {
|
||||
updates.push({ id: d.id, url });
|
||||
found++;
|
||||
processedIds.push(d.id);
|
||||
console.log("✅");
|
||||
} else {
|
||||
processedIds.push(d.id); // Mark as attempted even if not found
|
||||
console.log("❌");
|
||||
}
|
||||
|
||||
// Flush to DB
|
||||
if (updates.length >= DB_FLUSH_BATCH) {
|
||||
await raw.batch(
|
||||
updates.map((u) => ({
|
||||
sql: "UPDATE diseases SET image_url = ?, updated_at = datetime() WHERE id = ?",
|
||||
args: [u.url, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
console.log(` → Flushed ${updates.length} to DB`);
|
||||
updates = [];
|
||||
}
|
||||
|
||||
// Save state every 50
|
||||
if ((i + 1) % 50 === 0) {
|
||||
saveState(processedIds);
|
||||
}
|
||||
|
||||
// Rate limit (even for DDG to be polite)
|
||||
await new Promise((r) => setTimeout(r, ddgMode ? 500 : BRAVE_DELAY));
|
||||
}
|
||||
|
||||
// Final flush
|
||||
if (updates.length > 0) {
|
||||
await raw.batch(
|
||||
updates.map((u) => ({
|
||||
sql: "UPDATE diseases SET image_url = ?, updated_at = datetime() WHERE id = ?",
|
||||
args: [u.url, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
console.log(` → Flushed ${updates.length} to DB`);
|
||||
}
|
||||
|
||||
saveState(processedIds);
|
||||
raw.close();
|
||||
|
||||
// Final report
|
||||
const finalList = await db
|
||||
.select({ id: diseases.id, name: diseases.name, imageUrl: diseases.imageUrl })
|
||||
.from(diseases)
|
||||
.all();
|
||||
const w = finalList.filter((d) => d.imageUrl);
|
||||
const wo = finalList.filter((d) => !d.imageUrl);
|
||||
|
||||
console.log(`\n${"═".repeat(50)}`);
|
||||
console.log(`📊 BRAVE IMAGE SEARCH COMPLETE`);
|
||||
console.log(`${"═".repeat(50)}`);
|
||||
console.log(` Processed: ${pending.length}`);
|
||||
console.log(` Found this run: ${found - totalFound}`);
|
||||
console.log(` Total with images: ${w.length}/${finalList.length}`);
|
||||
console.log(` Still missing: ${wo.length}`);
|
||||
console.log(` Brave keys used: ${currentKeyIndex + 1}`);
|
||||
console.log(` Calls on current key: ${callsThisKey}`);
|
||||
console.log(` DuckDuckGo mode: ${ddgMode}`);
|
||||
|
||||
if (wo.length > 0) {
|
||||
const rp = resolve(__dirname, ".disease-image-review-needed.md");
|
||||
let report = "# Disease Images - Still Missing\n\n";
|
||||
report += `Generated: ${new Date().toISOString()}\n\n`;
|
||||
report += `## Summary\n\n`;
|
||||
report += `- Total: ${finalList.length}\n`;
|
||||
report += `- With images: ${w.length}\n`;
|
||||
report += `- Still missing: ${wo.length}\n\n`;
|
||||
report += `## Missing Diseases\n\n`;
|
||||
for (const d of wo) {
|
||||
report += `- ${d.name} (\`${d.id}\`)\n`;
|
||||
}
|
||||
writeFileSync(rp, report, "utf-8");
|
||||
console.log(`\n📝 Report: ${rp}`);
|
||||
} else {
|
||||
console.log("\n✅ ALL diseases now have images!");
|
||||
}
|
||||
|
||||
closeDb();
|
||||
console.log("\n");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\n❌", err);
|
||||
process.exit(1);
|
||||
});
|
||||
152
scripts/fill-brave-images.ts
Normal file
152
scripts/fill-brave-images.ts
Normal file
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* fill-brave-images.ts — Brave-only pass for remaining disease images.
|
||||
*
|
||||
* Runs at 1 request/sec (Brave rate limit).
|
||||
* Updates diseases.json and Turso DB.
|
||||
*
|
||||
* Usage: cd apps/web && npx tsx scripts/fill-brave-images.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv"; dotenv.config({ path: resolve(__dirname, "../.env.local") });
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import { createClient } from "@libsql/client";
|
||||
import { closeDb } from "../src/lib/db/index";
|
||||
|
||||
const DISEASES_JSON = resolve(__dirname, "../src/data/diseases.json");
|
||||
const BRAVE_KEY = process.env.BRAVE_API_KEY ?? "";
|
||||
|
||||
interface DiseaseSeed {
|
||||
id: string;
|
||||
plantId: string;
|
||||
name: string;
|
||||
scientificName: string;
|
||||
imageUrl?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
function load(): DiseaseSeed[] {
|
||||
return JSON.parse(readFileSync(DISEASES_JSON, "utf-8")) as DiseaseSeed[];
|
||||
}
|
||||
|
||||
async function searchBraveImage(query: string): Promise<string | null> {
|
||||
const url = new URL("https://api.search.brave.com/res/v1/images/search");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("count", "3");
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
const res = await fetch(url.toString(), {
|
||||
headers: { "X-Subscription-Token": BRAVE_KEY, Accept: "application/json" },
|
||||
});
|
||||
if (res.status === 429) {
|
||||
await new Promise((r) => setTimeout(r, 5000 * 2 ** attempt));
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as {
|
||||
results?: Array<{ url: string; thumbnail?: { src?: string } }>;
|
||||
};
|
||||
const results = data?.results ?? [];
|
||||
if (results.length === 0) return null;
|
||||
|
||||
// Prefer non-stock direct-looking images
|
||||
for (const r of results) {
|
||||
const src = r.thumbnail?.src ?? r.url;
|
||||
if (src && !/(dreamstime|shutterstock|alamy|istock|123rf)/i.test(src)) return src;
|
||||
}
|
||||
return results[0].thumbnail?.src ?? results[0].url;
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("\n🔍 Brave Image Search — remaining disease images\n");
|
||||
|
||||
if (!BRAVE_KEY) {
|
||||
console.log("❌ No BRAVE_API_KEY in .env.local\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const diseases = load();
|
||||
const pending = diseases.filter((d) => !d.imageUrl);
|
||||
console.log(`📋 ${pending.length} diseases need images\n`);
|
||||
|
||||
let found = 0;
|
||||
|
||||
for (let i = 0; i < pending.length; i++) {
|
||||
const d = pending[i];
|
||||
const plant = diseases.find((p) => p.id === d.plantId);
|
||||
const plantName = plant?.name ?? d.plantId;
|
||||
const query = `${d.name} ${plantName} plant disease symptom`;
|
||||
|
||||
process.stdout.write(` [${String(i + 1).padStart(2, " ")}/${pending.length}] ${d.name.padEnd(35)} `);
|
||||
|
||||
const url = await searchBraveImage(query);
|
||||
if (url) {
|
||||
d.imageUrl = url;
|
||||
found++;
|
||||
console.log(`✅`);
|
||||
} else {
|
||||
console.log(`❌`);
|
||||
}
|
||||
|
||||
// 1 req/sec rate limit
|
||||
await new Promise((r) => setTimeout(r, 1100));
|
||||
}
|
||||
|
||||
// Write updated JSON
|
||||
writeFileSync(DISEASES_JSON, JSON.stringify(diseases, null, 2) + "\n", "utf-8");
|
||||
console.log(`\n✅ diseases.json updated: ${found}/${pending.length} images found\n`);
|
||||
|
||||
// Update DB
|
||||
try {
|
||||
const dbUrl = process.env.DATABASE_URL;
|
||||
const dbToken = process.env.DATABASE_TOKEN;
|
||||
if (dbUrl && dbToken) {
|
||||
const raw = createClient({ url: dbUrl, authToken: dbToken });
|
||||
const updates = pending.filter((d) => d.imageUrl);
|
||||
for (let i = 0; i < updates.length; i += 50) {
|
||||
await raw.batch(
|
||||
updates.slice(i, i + 50).map((d) => ({
|
||||
sql: "UPDATE diseases SET image_url = ? WHERE id = ?",
|
||||
args: [d.imageUrl!, d.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
}
|
||||
raw.close();
|
||||
console.log(`✅ Turso DB updated: ${updates.length} rows`);
|
||||
} else {
|
||||
console.log("⏭️ Skipping DB — no DATABASE_URL/TOKEN");
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(` ⚠️ DB: ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
|
||||
// Summary
|
||||
const finalDiseases = JSON.parse(readFileSync(DISEASES_JSON, "utf-8")) as DiseaseSeed[];
|
||||
const stillMissing = finalDiseases.filter((d) => !d.imageUrl);
|
||||
console.log(`\n${"═".repeat(50)}`);
|
||||
console.log(`📊 FINAL: ${finalDiseases.length} total`);
|
||||
console.log(` With images: ${finalDiseases.length - stillMissing.length}`);
|
||||
console.log(` Still missing: ${stillMissing.length}`);
|
||||
if (stillMissing.length > 0) {
|
||||
console.log(`\nStill need human curation:`);
|
||||
for (const d of stillMissing) {
|
||||
console.log(` ❌ ${d.name} (${d.id})`);
|
||||
}
|
||||
}
|
||||
console.log(`${"═".repeat(50)}\n`);
|
||||
|
||||
closeDb();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\n❌ Fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
268
scripts/fill-ddg-images.ts
Normal file
268
scripts/fill-ddg-images.ts
Normal file
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* fill-ddg-images.ts — DuckDuckGo Image Search for remaining disease images.
|
||||
*
|
||||
* No API key needed. Searches DuckDuckGo Images API for each disease
|
||||
* without an image and updates the Turso DB.
|
||||
*
|
||||
* Prioritizes by severity (critical → high → moderate → low).
|
||||
* Runs at 1 request/sec to be polite to DuckDuckGo.
|
||||
* Resumable via state file (scripts/.ddg-progress.json).
|
||||
*
|
||||
* Usage:
|
||||
* cd apps/web && npx tsx scripts/fill-ddg-images.ts
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
|
||||
// Load .env.development for DB creds
|
||||
const envPath = resolve(__dirname, "../.env.development");
|
||||
try {
|
||||
const env = readFileSync(envPath, "utf-8");
|
||||
for (const line of env.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed && !trimmed.startsWith("#")) {
|
||||
const eqIdx = trimmed.indexOf("=");
|
||||
if (eqIdx > 0) {
|
||||
const key = trimmed.slice(0, eqIdx).trim();
|
||||
const val = trimmed.slice(eqIdx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
import { getDb, closeDb } from "../src/lib/db/index";
|
||||
import { diseases } from "../src/lib/db/schema";
|
||||
import { createClient } from "@libsql/client";
|
||||
import { sql } from "drizzle-orm";
|
||||
|
||||
// DuckDuckGo
|
||||
import { imageSearch } from "@mudbill/duckduckgo-images-api";
|
||||
|
||||
interface DiseaseRow {
|
||||
id: string;
|
||||
name: string;
|
||||
scientificName: string;
|
||||
severity: string;
|
||||
plantId: string;
|
||||
}
|
||||
|
||||
// ─── Config ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const POLITE_DELAY = 800; // ms between calls
|
||||
const DB_FLUSH_BATCH = 50;
|
||||
const STATE_FILE = resolve(__dirname, ".ddg-progress.json");
|
||||
|
||||
interface RunState {
|
||||
processedIds: string[];
|
||||
totalFound: number;
|
||||
}
|
||||
|
||||
function loadState(): RunState | null {
|
||||
try {
|
||||
return JSON.parse(readFileSync(STATE_FILE, "utf-8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function saveState(processedIds: string[], totalFound: number) {
|
||||
writeFileSync(STATE_FILE, JSON.stringify({ processedIds, totalFound }, null, 2), "utf-8");
|
||||
}
|
||||
|
||||
// ─── DuckDuckGo Search ───────────────────────────────────────────────────────
|
||||
|
||||
async function searchImage(query: string): Promise<string | null> {
|
||||
try {
|
||||
const results = await imageSearch({ query, safe: true, iterations: 1, retries: 2 });
|
||||
if (!results || results.length === 0) return null;
|
||||
|
||||
// Prefer non-stock images
|
||||
for (const r of results) {
|
||||
if (r.image && !/(dreamstime|shutterstock|alamy|istock|123rf)/i.test(r.image)) {
|
||||
return r.image;
|
||||
}
|
||||
}
|
||||
return results[0].image || results[0].thumbnail || null;
|
||||
} catch {
|
||||
// DuckDuckGo may block or timeout; silently skip
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
console.log("\n🦆 DuckDuckGo Disease Image Filler\n");
|
||||
|
||||
const db = getDb();
|
||||
|
||||
// Load state
|
||||
const state = loadState();
|
||||
const processedSet = new Set(state?.processedIds || []);
|
||||
const totalFoundPrev = state?.totalFound ?? 0;
|
||||
|
||||
// Get all diseases that still need images
|
||||
const allDiseases = (await db
|
||||
.select({
|
||||
id: diseases.id,
|
||||
name: diseases.name,
|
||||
scientificName: diseases.scientificName,
|
||||
severity: diseases.severity,
|
||||
plantId: diseases.plantId,
|
||||
})
|
||||
.from(diseases)
|
||||
.where(sql`(image_url IS NULL OR image_url = '')`)
|
||||
.all()) as DiseaseRow[];
|
||||
|
||||
console.log(`📋 ${allDiseases.length} diseases need images\n`);
|
||||
|
||||
if (allDiseases.length === 0) {
|
||||
console.log("✅ All diseases already have images!\n");
|
||||
closeDb();
|
||||
return;
|
||||
}
|
||||
|
||||
// Sort by severity: critical > high > moderate > low
|
||||
const severityOrder: Record<string, number> = { critical: 0, high: 1, moderate: 2, low: 3 };
|
||||
allDiseases.sort((a, b) => (severityOrder[a.severity] ?? 99) - (severityOrder[b.severity] ?? 99));
|
||||
|
||||
// Filter out already-processed
|
||||
const pending = allDiseases.filter((d) => !processedSet.has(d.id));
|
||||
|
||||
console.log(
|
||||
`📊 Remaining: critical=${allDiseases.filter((d) => d.severity === "critical" && !processedSet.has(d.id)).length}, ` +
|
||||
`high=${allDiseases.filter((d) => d.severity === "high" && !processedSet.has(d.id)).length}, ` +
|
||||
`moderate=${allDiseases.filter((d) => d.severity === "moderate" && !processedSet.has(d.id)).length}, ` +
|
||||
`low=${allDiseases.filter((d) => d.severity === "low" && !processedSet.has(d.id)).length}\n`,
|
||||
);
|
||||
|
||||
if (pending.length === 0) {
|
||||
console.log("✅ All remaining diseases already attempted\n");
|
||||
closeDb();
|
||||
return;
|
||||
}
|
||||
|
||||
const raw = createClient({
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
});
|
||||
|
||||
const processedIds: string[] = state?.processedIds ?? [];
|
||||
let found = totalFoundPrev;
|
||||
let updates: Array<{ id: string; url: string }> = [];
|
||||
|
||||
for (let i = 0; i < pending.length; i++) {
|
||||
const d = pending[i];
|
||||
const sev = d.severity.padEnd(8);
|
||||
|
||||
// Build search query — "[disease] on [plant]" phrasing for better specificity
|
||||
const plantName = d.plantId.replace(/-/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
const query1 = `${d.name} on ${plantName} plant disease`;
|
||||
const query2 = `${d.scientificName || d.name} on ${plantName} disease`;
|
||||
const query3 = `${d.name} plant disease ${plantName}`;
|
||||
const query4 = `${d.name} plant`;
|
||||
const query5 = `${d.name} symptom`;
|
||||
|
||||
process.stdout.write(
|
||||
` [${String(i + 1).padStart(4)}/${pending.length}] [${sev}] ${d.name.substring(0, 42).padEnd(44)} `,
|
||||
);
|
||||
|
||||
// Try queries in order until we get a result
|
||||
let url: string | null = null;
|
||||
for (const q of [query1, query2, query3, query4, query5]) {
|
||||
url = await searchImage(q);
|
||||
if (url) break;
|
||||
}
|
||||
|
||||
if (url) {
|
||||
updates.push({ id: d.id, url });
|
||||
found++;
|
||||
processedIds.push(d.id);
|
||||
console.log("✅");
|
||||
} else {
|
||||
processedIds.push(d.id);
|
||||
console.log("❌");
|
||||
}
|
||||
|
||||
// Flush to DB in batches
|
||||
if (updates.length >= DB_FLUSH_BATCH) {
|
||||
await raw.batch(
|
||||
updates.map((u) => ({
|
||||
sql: "UPDATE diseases SET image_url = ?, updated_at = datetime() WHERE id = ?",
|
||||
args: [u.url, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
console.log(` → Flushed ${updates.length} to DB`);
|
||||
updates = [];
|
||||
}
|
||||
|
||||
// Save state every 50
|
||||
if ((i + 1) % 50 === 0) {
|
||||
saveState(processedIds, found);
|
||||
}
|
||||
|
||||
// Be polite — 1 req/sec
|
||||
await new Promise((r) => setTimeout(r, POLITE_DELAY));
|
||||
}
|
||||
|
||||
// Final flush
|
||||
if (updates.length > 0) {
|
||||
await raw.batch(
|
||||
updates.map((u) => ({
|
||||
sql: "UPDATE diseases SET image_url = ?, updated_at = datetime() WHERE id = ?",
|
||||
args: [u.url, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
console.log(` → Flushed ${updates.length} to DB`);
|
||||
}
|
||||
|
||||
saveState(processedIds, found);
|
||||
raw.close();
|
||||
|
||||
// Final report
|
||||
const finalList = await db
|
||||
.select({ id: diseases.id, name: diseases.name, imageUrl: diseases.imageUrl })
|
||||
.from(diseases)
|
||||
.all();
|
||||
const w = finalList.filter((d) => d.imageUrl);
|
||||
const wo = finalList.filter((d) => !d.imageUrl);
|
||||
|
||||
console.log(`\n${"═".repeat(50)}`);
|
||||
console.log(`🦆 DUCKDUCKGO SEARCH COMPLETE`);
|
||||
console.log(`${"═".repeat(50)}`);
|
||||
console.log(` Processed: ${pending.length}`);
|
||||
console.log(` Found this run: ${found - totalFoundPrev}`);
|
||||
console.log(` Total with images: ${w.length}/${finalList.length}`);
|
||||
console.log(` Still missing: ${wo.length}`);
|
||||
|
||||
if (wo.length > 0) {
|
||||
const reportPath = resolve(__dirname, ".ddg-image-review-needed.md");
|
||||
let report = "# Disease Images - Still Missing (DDG)\n\n";
|
||||
report += `Generated: ${new Date().toISOString()}\n\n`;
|
||||
report += `## Summary\n\n`;
|
||||
report += `- Total: ${finalList.length}\n`;
|
||||
report += `- With images: ${w.length}\n`;
|
||||
report += `- Still missing: ${wo.length}\n\n`;
|
||||
report += `## Missing Diseases\n\n`;
|
||||
for (const d of wo) {
|
||||
report += `- ${d.name} (\`${d.id}\`)\n`;
|
||||
}
|
||||
writeFileSync(reportPath, report, "utf-8");
|
||||
console.log(`\n📝 Missing report: ${reportPath}`);
|
||||
} else {
|
||||
console.log("\n✅ ALL diseases now have images!");
|
||||
}
|
||||
|
||||
closeDb();
|
||||
console.log();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\n❌ Fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
440
scripts/fill-disease-images.ts
Normal file
440
scripts/fill-disease-images.ts
Normal file
@@ -0,0 +1,440 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* fill-disease-images.ts — Three-stage disease image pipeline
|
||||
*
|
||||
* For every disease without an imageUrl, tries:
|
||||
* Stage 1 — Wikipedia search → pageimages
|
||||
* Stage 2 — Wikimedia Commons search
|
||||
* Stage 3 — Brave Image Search API (fallback, 1 req/sec, 2000/mo)
|
||||
*
|
||||
* Updates both diseases.json (seed) and the Turso DB.
|
||||
* Flags anything found only via Brave for human review.
|
||||
*
|
||||
* Usage: cd apps/web && npx tsx scripts/fill-disease-images.ts
|
||||
*/
|
||||
|
||||
import "dotenv/config";
|
||||
import { readFileSync, writeFileSync, existsSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import { createClient } from "@libsql/client";
|
||||
import { closeDb } from "../src/lib/db/index";
|
||||
|
||||
// ─── Types & Config ──────────────────────────────────────────────────────────
|
||||
|
||||
interface DiseaseSeed {
|
||||
id: string;
|
||||
plantId: string;
|
||||
name: string;
|
||||
scientificName: string;
|
||||
commonName?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
interface ImageResult {
|
||||
url: string;
|
||||
source: "wikipedia" | "commons" | "brave" | "missing";
|
||||
quality: "good" | "fallback" | "missing";
|
||||
}
|
||||
|
||||
const DISEASES_JSON = resolve(__dirname, "../src/data/diseases.json");
|
||||
const RESULTS_FILE = resolve(__dirname, ".image-results.json");
|
||||
const REPORT_FILE = resolve(__dirname, ".image-review-needed.md");
|
||||
|
||||
const WIKI_API = "https://en.wikipedia.org/w/api.php";
|
||||
const COMMONS_API = "https://commons.wikimedia.org/w/api.php";
|
||||
const BRAVE_KEY = process.env.BRAVE_API_KEY ?? "";
|
||||
const BRAVE_DELAY = 1100;
|
||||
const MAX_BRAVE = 2000;
|
||||
const UA = "PlantHealthKB/1.0 (plant-disease-id)";
|
||||
const ORIGIN = "*";
|
||||
|
||||
let braveCount = 0;
|
||||
|
||||
// ─── Wikipedia Stage ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Search Wikipedia and get thumbnails in ONE API call using generator=search.
|
||||
* Returns first thumbnail found, or null.
|
||||
*/
|
||||
async function wikiSearchAndThumb(query: string): Promise<string | null> {
|
||||
const params = new URLSearchParams({
|
||||
action: "query",
|
||||
generator: "search",
|
||||
gsrsearch: query,
|
||||
gsrlimit: "5",
|
||||
prop: "pageimages",
|
||||
pithumbsize: "600",
|
||||
format: "json",
|
||||
origin: ORIGIN,
|
||||
});
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
const res = await fetchWithTimeout(`${WIKI_API}?${params}`, {
|
||||
headers: { "User-Agent": UA },
|
||||
});
|
||||
if (res.status === 429) {
|
||||
await delay(3000 * 2 ** attempt);
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as {
|
||||
query?: { pages?: Record<string, { thumbnail?: { source: string } }> };
|
||||
};
|
||||
const pages = data?.query?.pages;
|
||||
if (!pages) return null;
|
||||
for (const [, p] of Object.entries(pages)) {
|
||||
const src = (p as { thumbnail?: { source: string } })?.thumbnail?.source;
|
||||
if (src) return src;
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
await delay(2000);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to find a Wikipedia image for a disease.
|
||||
* Uses generator=search which combines search + thumbnails in one call.
|
||||
*/
|
||||
async function wikiStage(d: DiseaseSeed, plantName: string): Promise<string | null> {
|
||||
// Try 1: disease name + plant name (most specific)
|
||||
return wikiSearchAndThumb(`"${d.name}" ${plantName}`);
|
||||
}
|
||||
|
||||
// ─── Commons Stage ───────────────────────────────────────────────────────────
|
||||
|
||||
/** Fetch with timeout. Aborts after `ms` milliseconds. */
|
||||
async function fetchWithTimeout(url: string, opts: RequestInit, ms = 15000): Promise<Response> {
|
||||
const ctrl = new AbortController();
|
||||
const timer = setTimeout(() => ctrl.abort(), ms);
|
||||
try {
|
||||
const res = await fetch(url, { ...opts, signal: ctrl.signal });
|
||||
return res;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function commonsSearchAndThumb(query: string): Promise<string | null> {
|
||||
const params = new URLSearchParams({
|
||||
action: "query",
|
||||
list: "search",
|
||||
srsearch: query,
|
||||
srnamespace: "6",
|
||||
srlimit: "5",
|
||||
format: "json",
|
||||
origin: ORIGIN,
|
||||
});
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
const res = await fetchWithTimeout(`${COMMONS_API}?${params}`, {
|
||||
headers: { "User-Agent": UA },
|
||||
});
|
||||
if (res.status === 429) {
|
||||
await delay(3000 * 2 ** attempt);
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as {
|
||||
query?: { search?: Array<{ pageid: number; title: string }> };
|
||||
};
|
||||
const hits = data?.query?.search ?? [];
|
||||
if (hits.length === 0) return null;
|
||||
|
||||
// Batch-fetch imageinfo for all found page IDs
|
||||
const pageids = hits.map((h) => h.pageid).join("|");
|
||||
const imgParams = new URLSearchParams({
|
||||
action: "query",
|
||||
pageids,
|
||||
prop: "imageinfo",
|
||||
iiprop: "url",
|
||||
iiurlwidth: "600",
|
||||
format: "json",
|
||||
origin: ORIGIN,
|
||||
});
|
||||
|
||||
const imgRes = await fetchWithTimeout(`${COMMONS_API}?${imgParams}`, {
|
||||
headers: { "User-Agent": UA },
|
||||
});
|
||||
if (!imgRes.ok) return null;
|
||||
const imgData = (await imgRes.json()) as {
|
||||
query?: { pages?: Record<string, unknown> };
|
||||
};
|
||||
const imgPages = imgData?.query?.pages;
|
||||
if (!imgPages) return null;
|
||||
|
||||
for (const [, pg] of Object.entries(imgPages)) {
|
||||
const p = pg as Record<string, unknown>;
|
||||
const info = (p.imageinfo as Array<Record<string, string>> | undefined)?.[0];
|
||||
if (info?.thumburl) return info.thumburl as string;
|
||||
if (info?.url) return info.url as string;
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
await delay(2000);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function commonsStage(d: DiseaseSeed, plantName: string): Promise<string | null> {
|
||||
let q: string;
|
||||
if (d.scientificName && !d.scientificName.includes("spp.") && !d.scientificName.includes("/")) {
|
||||
q = `${d.scientificName} ${plantName}`;
|
||||
} else {
|
||||
q = `${d.name} ${plantName} disease`;
|
||||
}
|
||||
|
||||
const url = await commonsSearchAndThumb(q);
|
||||
return url ?? null;
|
||||
}
|
||||
|
||||
// ─── Brave Stage ─────────────────────────────────────────────────────────────
|
||||
|
||||
async function braveStage(d: DiseaseSeed, plantName: string): Promise<string | null> {
|
||||
if (!BRAVE_KEY || braveCount >= MAX_BRAVE) return null;
|
||||
|
||||
const url = new URL("https://api.search.brave.com/res/v1/images/search");
|
||||
url.searchParams.set("q", `${d.name} ${plantName} plant disease symptom`);
|
||||
url.searchParams.set("count", "5");
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
const res = await fetchWithTimeout(url.toString(), {
|
||||
headers: { "X-Subscription-Token": BRAVE_KEY, Accept: "application/json" },
|
||||
});
|
||||
if (res.status === 429) {
|
||||
await delay(5000 * 2 ** attempt);
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) return null;
|
||||
braveCount++;
|
||||
const data = (await res.json()) as {
|
||||
results?: Array<{ url: string; thumbnail?: { src?: string } }>;
|
||||
};
|
||||
const results = data?.results ?? [];
|
||||
if (results.length === 0) return null;
|
||||
|
||||
// Prefer non-stock thumbnails
|
||||
for (const r of results) {
|
||||
const src = r.thumbnail?.src ?? r.url;
|
||||
if (src && !src.includes("dreamstime") && !src.includes("shutterstock") &&
|
||||
!src.includes("alamy") && !src.includes("istock") && !src.includes("123rf")) {
|
||||
return src;
|
||||
}
|
||||
}
|
||||
return results[0].thumbnail?.src ?? results[0].url;
|
||||
} catch {
|
||||
await delay(2000);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
function loadDiseases(): DiseaseSeed[] {
|
||||
return JSON.parse(readFileSync(DISEASES_JSON, "utf-8")) as DiseaseSeed[];
|
||||
}
|
||||
|
||||
function getPlantName(diseases: DiseaseSeed[], diseaseId: string): string {
|
||||
const plant = diseases.find((p) => p.id === diseaseId);
|
||||
return plant?.commonName ?? plant?.name ?? diseaseId;
|
||||
}
|
||||
|
||||
// ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
console.log("\n🔍 Plant Disease Image Filler\n");
|
||||
|
||||
const diseases = loadDiseases();
|
||||
console.log(`📋 ${diseases.length} diseases loaded\n`);
|
||||
|
||||
// Load existing results
|
||||
let results: Record<string, ImageResult> = {};
|
||||
if (existsSync(RESULTS_FILE)) {
|
||||
try { results = JSON.parse(readFileSync(RESULTS_FILE, "utf-8")); } catch { /* fresh */ }
|
||||
}
|
||||
|
||||
const pending = diseases.filter((d) => {
|
||||
if ((d.imageUrl as string)?.length) return false;
|
||||
return !results[d.id];
|
||||
});
|
||||
|
||||
if (pending.length === 0) {
|
||||
console.log("✅ All done\n");
|
||||
await applyResults(diseases, results);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`⏳ ${pending.length} need images\n`);
|
||||
|
||||
// ── Stage 1: Wikipedia ──────────────────────────────────────────────
|
||||
const s1 = pending.filter((d) => !results[d.id]);
|
||||
let s1ok = 0;
|
||||
console.log("─── Wikipedia ───\n");
|
||||
|
||||
for (let i = 0; i < s1.length; i++) {
|
||||
const d = s1[i];
|
||||
const plantName = getPlantName(diseases, d.plantId);
|
||||
const url = await wikiStage(d, plantName);
|
||||
if (url) {
|
||||
results[d.id] = { url, source: "wikipedia", quality: "good" };
|
||||
s1ok++;
|
||||
}
|
||||
const pct = ((i + 1) / s1.length * 100).toFixed(0);
|
||||
process.stdout.write(` [${pct}% ${i + 1}/${s1.length}] ${d.name.substring(0, 40).padEnd(42)} ${url ? "✅" : "⏭️"}\n`);
|
||||
if ((i + 1) % 25 === 0) writeFileSync(RESULTS_FILE, JSON.stringify(results, null, 2));
|
||||
}
|
||||
|
||||
writeFileSync(RESULTS_FILE, JSON.stringify(results, null, 2));
|
||||
console.log(`\n → ${s1ok}/${s1.length} found\n`);
|
||||
|
||||
// ── Stage 2: Commons ─────────────────────────────────────────────────
|
||||
const s2 = pending.filter((d) => !results[d.id]);
|
||||
let s2ok = 0;
|
||||
|
||||
if (s2.length > 0) {
|
||||
console.log("─── Wikimedia Commons ───\n");
|
||||
for (let i = 0; i < s2.length; i++) {
|
||||
const d = s2[i];
|
||||
const plantName = getPlantName(diseases, d.plantId);
|
||||
let url: string | null = null;
|
||||
try {
|
||||
const result = await Promise.race([
|
||||
commonsStage(d, plantName),
|
||||
new Promise<null>((_, reject) => setTimeout(() => reject(new Error("timeout")), 25000)),
|
||||
]);
|
||||
url = result;
|
||||
} catch { /* timeout */ }
|
||||
if (url) {
|
||||
results[d.id] = { url, source: "commons", quality: "good" };
|
||||
s2ok++;
|
||||
}
|
||||
const pct = ((i + 1) / s2.length * 100).toFixed(0);
|
||||
process.stdout.write(` [${pct}% ${i + 1}/${s2.length}] ${d.name.substring(0, 40).padEnd(42)} ${url ? "✅" : "⏭️"}\n`);
|
||||
|
||||
if ((i + 1) % 10 === 0) writeFileSync(RESULTS_FILE, JSON.stringify(results, null, 2));
|
||||
}
|
||||
writeFileSync(RESULTS_FILE, JSON.stringify(results, null, 2));
|
||||
console.log(`\n → ${s2ok}/${s2.length} found\n`);
|
||||
}
|
||||
|
||||
// ── Stage 3: Brave ───────────────────────────────────────────────────
|
||||
const s3 = pending.filter((d) => !results[d.id]);
|
||||
let s3ok = 0;
|
||||
|
||||
if (s3.length > 0 && BRAVE_KEY) {
|
||||
console.log("─── Brave Image Search ───\n");
|
||||
for (const d of s3) {
|
||||
if (braveCount >= MAX_BRAVE) {
|
||||
results[d.id] = { url: "", source: "missing", quality: "missing" };
|
||||
continue;
|
||||
}
|
||||
const plantName = getPlantName(diseases, d.plantId);
|
||||
const url = await braveStage(d, plantName);
|
||||
if (url) {
|
||||
results[d.id] = { url, source: "brave", quality: "fallback" };
|
||||
s3ok++;
|
||||
process.stdout.write(` ✅ ${d.name}\n`);
|
||||
} else {
|
||||
results[d.id] = { url: "", source: "missing", quality: "missing" };
|
||||
process.stdout.write(` ❌ ${d.name}\n`);
|
||||
}
|
||||
await delay(BRAVE_DELAY);
|
||||
}
|
||||
writeFileSync(RESULTS_FILE, JSON.stringify(results, null, 2));
|
||||
console.log(`\n → ${s3ok}/${s3.length} found via Brave\n`);
|
||||
} else if (s3.length > 0) {
|
||||
console.log("─── Brave Image Search ─── → skipped (no key)\n");
|
||||
for (const d of s3) results[d.id] = { url: "", source: "missing", quality: "missing" };
|
||||
}
|
||||
|
||||
// ── Apply ───────────────────────────────────────────────────────────
|
||||
await applyResults(diseases, results);
|
||||
|
||||
// ── Report ──────────────────────────────────────────────────────────
|
||||
const good = Object.values(results).filter((r) => r.quality === "good").length;
|
||||
const fallback = Object.values(results).filter((r) => r.quality === "fallback").length;
|
||||
const missing = Object.values(results).filter((r) => r.quality === "missing").length;
|
||||
|
||||
let report = `# Disease Images — Human Review Needed\n\n`;
|
||||
report += `Generated: ${new Date().toISOString()}\n\n`;
|
||||
|
||||
for (const [label, ids, type] of [
|
||||
["Fallback (Brave)", Object.entries(results).filter(([, r]) => r.quality === "fallback").map(([id]) => id), "fallback"],
|
||||
["Missing", Object.entries(results).filter(([, r]) => r.quality === "missing").map(([id]) => id), "missing"],
|
||||
] as const) {
|
||||
if (ids.length === 0) continue;
|
||||
report += `## ${type === "fallback" ? "⚠️" : "🚫"} ${label}\n\n`;
|
||||
for (const id of ids) {
|
||||
const d = diseases.find((x) => x.id === id);
|
||||
const r = results[id];
|
||||
report += `- **${d?.name ?? id}** (${d?.scientificName ?? ""}) on \`${d?.plantId ?? ""}\``;
|
||||
if (r?.url) report += `\n ${r.url}`;
|
||||
report += `\n\n`;
|
||||
}
|
||||
}
|
||||
|
||||
if (good === diseases.length) report += `## ✅ All images found!\n`;
|
||||
writeFileSync(REPORT_FILE, report, "utf-8");
|
||||
console.log(`📝 Review report: ${REPORT_FILE}`);
|
||||
|
||||
console.log(`\n${"═".repeat(50)}`);
|
||||
console.log(`📊 Total: ${diseases.length} Good: ${good} Fallback: ${fallback} Missing: ${missing}`);
|
||||
console.log(` Brave calls: ${braveCount}`);
|
||||
console.log(`${"═".repeat(50)}\n`);
|
||||
|
||||
closeDb();
|
||||
}
|
||||
|
||||
// ─── Apply results to JSON + DB ──────────────────────────────────────────────
|
||||
|
||||
async function applyResults(diseases: DiseaseSeed[], results: Record<string, ImageResult>) {
|
||||
const urlMap = new Map(
|
||||
Object.entries(results).filter(([id, r]) => r.url.length > 0 && diseases.some((d) => d.id === id)),
|
||||
);
|
||||
if (urlMap.size === 0) return console.log("⏭️ No images to apply");
|
||||
|
||||
// JSON
|
||||
let n = 0;
|
||||
const updated = diseases.map((d) => {
|
||||
const img = urlMap.get(d.id);
|
||||
if (img) { n++; return { ...d, imageUrl: img.url, imageQuality: img.quality }; }
|
||||
return d;
|
||||
});
|
||||
writeFileSync(DISEASES_JSON, JSON.stringify(updated, null, 2) + "\n");
|
||||
console.log(`✅ diseases.json: ${n} images`);
|
||||
|
||||
// DB
|
||||
try {
|
||||
const dbUrl = process.env.DATABASE_URL;
|
||||
const dbToken = process.env.DATABASE_TOKEN;
|
||||
if (!dbUrl || !dbToken) return console.log(" ⏭️ DB: no DATABASE_URL/TOKEN");
|
||||
const raw = createClient({ url: dbUrl, authToken: dbToken });
|
||||
const entries = Array.from(urlMap.entries());
|
||||
for (let i = 0; i < entries.length; i += 50) {
|
||||
await raw.batch(
|
||||
entries.slice(i, i + 50).map(([id, img]) => ({
|
||||
sql: "UPDATE diseases SET image_url = ? WHERE id = ?",
|
||||
args: [img.url, id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
}
|
||||
raw.close();
|
||||
console.log(`✅ Turso DB: ${entries.length} rows`);
|
||||
} catch (err) {
|
||||
console.log(` ⚠️ DB: ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => { console.error("\n❌", err); process.exit(1); });
|
||||
301
scripts/fill-plant-images-v2.ts
Normal file
301
scripts/fill-plant-images-v2.ts
Normal file
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* fill-plant-images-v2.ts — Batch Wikipedia image fetch for remaining plants.
|
||||
*
|
||||
* Phase 1: Query 50 scientific names at a time via pageimages.
|
||||
* Phase 2: Query 50 common names at a time.
|
||||
* Phase 3: Search individually for stragglers.
|
||||
*
|
||||
* Usage: cd apps/web && npx tsx scripts/fill-plant-images-v2.ts
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
|
||||
// Load env
|
||||
const envPath = resolve(__dirname, "../.env.development");
|
||||
try {
|
||||
const env = readFileSync(envPath, "utf-8");
|
||||
for (const line of env.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed && !trimmed.startsWith("#")) {
|
||||
const eqIdx = trimmed.indexOf("=");
|
||||
if (eqIdx > 0) {
|
||||
const key = trimmed.slice(0, eqIdx).trim();
|
||||
const val = trimmed.slice(eqIdx + 1).trim();
|
||||
if (!process.env[key]) {
|
||||
process.env[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {}
|
||||
|
||||
import { getDb, closeDb } from "../src/lib/db/index";
|
||||
import { plants } from "../src/lib/db/schema";
|
||||
import { createClient } from "@libsql/client";
|
||||
import { sql } from "drizzle-orm";
|
||||
|
||||
const API = "https://en.wikipedia.org/w/api.php";
|
||||
const UA = "PlantHealthKB/1.0";
|
||||
const BATCH = 50;
|
||||
|
||||
interface PlantRow {
|
||||
id: string;
|
||||
commonName: string;
|
||||
scientificName: string;
|
||||
}
|
||||
|
||||
function clean(s: string): string {
|
||||
return s
|
||||
.replace(/[xX]/g, "x")
|
||||
.replace(/\s*spp\.?\s*/gi, "")
|
||||
.replace(/[.\u00d7']/g, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
async function fetchThumbs(titles: string[]): Promise<Map<string, string>> {
|
||||
if (titles.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
const p = new URLSearchParams({
|
||||
action: "query",
|
||||
titles: titles.join("|"),
|
||||
prop: "pageimages",
|
||||
pithumbsize: "400",
|
||||
redirects: "1",
|
||||
format: "json",
|
||||
});
|
||||
for (let a = 0; a < 3; a++) {
|
||||
try {
|
||||
const r = await fetch(API + "?" + p.toString(), {
|
||||
headers: { "User-Agent": UA },
|
||||
});
|
||||
if (r.status === 429) {
|
||||
await new Promise((rr) => setTimeout(rr, 5000 * Math.pow(2, a)));
|
||||
continue;
|
||||
}
|
||||
if (!r.ok) {
|
||||
return new Map();
|
||||
}
|
||||
const d = (await r.json()) as any;
|
||||
const pages = d?.query?.pages;
|
||||
if (!pages) {
|
||||
return new Map();
|
||||
}
|
||||
const m = new Map<string, string>();
|
||||
for (const [, pg] of Object.entries(pages)) {
|
||||
const p2 = pg as any;
|
||||
if (!p2.missing && p2.thumbnail?.source) {
|
||||
m.set(p2.title.toLowerCase(), p2.thumbnail.source);
|
||||
}
|
||||
}
|
||||
return m;
|
||||
} catch (e) {
|
||||
await new Promise((rr) => setTimeout(rr, 2000));
|
||||
}
|
||||
}
|
||||
return new Map();
|
||||
}
|
||||
|
||||
async function searchOne(query: string): Promise<string | null> {
|
||||
const p = new URLSearchParams({
|
||||
action: "query",
|
||||
generator: "search",
|
||||
gsrsearch: query,
|
||||
gsrlimit: "3",
|
||||
prop: "pageimages",
|
||||
pithumbsize: "400",
|
||||
format: "json",
|
||||
});
|
||||
for (let a = 0; a < 3; a++) {
|
||||
try {
|
||||
const r = await fetch(API + "?" + p.toString(), {
|
||||
headers: { "User-Agent": UA },
|
||||
});
|
||||
if (r.status === 429) {
|
||||
await new Promise((rr) => setTimeout(rr, 5000 * Math.pow(2, a)));
|
||||
continue;
|
||||
}
|
||||
if (!r.ok) {
|
||||
return null;
|
||||
}
|
||||
const d = (await r.json()) as any;
|
||||
const pages = d?.query?.pages;
|
||||
if (!pages) {
|
||||
return null;
|
||||
}
|
||||
for (const [, pg] of Object.entries(pages)) {
|
||||
const p2 = pg as any;
|
||||
if (p2.thumbnail?.source) {
|
||||
return p2.thumbnail.source;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
} catch (e) {
|
||||
await new Promise((rr) => setTimeout(rr, 2000));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function batchPhase(
|
||||
plants: PlantRow[],
|
||||
titleFn: (p: PlantRow) => string,
|
||||
label: string,
|
||||
dbClient: any,
|
||||
): Promise<PlantRow[]> {
|
||||
const remaining: PlantRow[] = [];
|
||||
const updates: Array<{ id: string; url: string }> = [];
|
||||
|
||||
for (let i = 0; i < plants.length; i += BATCH) {
|
||||
const chunk = plants.slice(i, i + BATCH);
|
||||
const titles = chunk.map(titleFn).filter((t) => t.length > 2);
|
||||
console.log(
|
||||
" [" +
|
||||
label +
|
||||
"] " +
|
||||
(i + 1) +
|
||||
"-" +
|
||||
Math.min(i + BATCH, plants.length) +
|
||||
"/" +
|
||||
plants.length +
|
||||
" ",
|
||||
);
|
||||
const imageMap = await fetchThumbs(titles);
|
||||
let n = 0;
|
||||
for (const pl of chunk) {
|
||||
const t = titleFn(pl).toLowerCase();
|
||||
const img = imageMap.get(t);
|
||||
if (img) {
|
||||
updates.push({ id: pl.id, url: img });
|
||||
n++;
|
||||
} else {
|
||||
remaining.push(pl);
|
||||
}
|
||||
}
|
||||
console.log(" found: " + n);
|
||||
if (updates.length >= 100) {
|
||||
await dbClient.batch(
|
||||
updates.map((u) => ({
|
||||
sql: "UPDATE plants SET image_url = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
args: [u.url, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
updates.length = 0;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
}
|
||||
|
||||
if (updates.length > 0) {
|
||||
await dbClient.batch(
|
||||
updates.map((u) => ({
|
||||
sql: "UPDATE plants SET image_url = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
args: [u.url, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
}
|
||||
|
||||
return remaining;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("\nPlant Image Filler v2\n");
|
||||
const db = getDb();
|
||||
const allPlants = (await db
|
||||
.select({
|
||||
id: plants.id,
|
||||
commonName: plants.commonName,
|
||||
scientificName: plants.scientificName,
|
||||
})
|
||||
.from(plants)
|
||||
.where(sql`(image_url IS NULL OR image_url = '')`)
|
||||
.all()) as PlantRow[];
|
||||
|
||||
console.log("Plants needing images: " + allPlants.length + "\n");
|
||||
if (allPlants.length === 0) {
|
||||
console.log("All plants have images!\n");
|
||||
closeDb();
|
||||
return;
|
||||
}
|
||||
|
||||
const raw = createClient({
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
});
|
||||
let found = 0;
|
||||
|
||||
// Phase 1: Scientific name
|
||||
console.log("--- Phase 1: Scientific names ---\n");
|
||||
let remaining = await batchPhase(allPlants, (p) => clean(p.scientificName), "sci", raw);
|
||||
|
||||
// Phase 2: Common name
|
||||
if (remaining.length > 0) {
|
||||
console.log("\n--- Phase 2: Common names (" + remaining.length + ") ---\n");
|
||||
remaining = await batchPhase(remaining, (p) => p.commonName, "common", raw);
|
||||
}
|
||||
|
||||
// Phase 3: Search
|
||||
if (remaining.length > 0) {
|
||||
console.log("\n--- Phase 3: Search (" + remaining.length + ") ---\n");
|
||||
for (let i = 0; i < remaining.length; i++) {
|
||||
const pl = remaining[i];
|
||||
const q = clean(pl.scientificName) + " " + pl.commonName;
|
||||
console.log(" [" + (i + 1) + "/" + remaining.length + "] " + pl.commonName);
|
||||
const img = await searchOne(q);
|
||||
if (img) {
|
||||
await raw.execute({
|
||||
sql: "UPDATE plants SET image_url = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
args: [img, pl.id],
|
||||
});
|
||||
found++;
|
||||
console.log(" OK");
|
||||
} else {
|
||||
console.log(" MISS");
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
}
|
||||
|
||||
raw.close();
|
||||
|
||||
// Report
|
||||
const finalList = await db
|
||||
.select({
|
||||
id: plants.id,
|
||||
commonName: plants.commonName,
|
||||
imageUrl: plants.imageUrl,
|
||||
})
|
||||
.from(plants)
|
||||
.all();
|
||||
const w = finalList.filter((p) => p.imageUrl);
|
||||
const wo = finalList.filter((p) => !p.imageUrl);
|
||||
|
||||
console.log("\n" + "=".repeat(50));
|
||||
console.log("FINAL: " + finalList.length + " plants");
|
||||
console.log(" With images: " + w.length);
|
||||
console.log(" Missing: " + wo.length);
|
||||
|
||||
if (wo.length > 0) {
|
||||
const rp = resolve(__dirname, ".plant-image-review-needed.md");
|
||||
let report = "# Plant Images - Still Missing\n\n";
|
||||
report += "Generated: " + new Date().toISOString() + "\n\n";
|
||||
report += "## Missing (" + wo.length + ")\n\n";
|
||||
for (const p of wo) {
|
||||
report += "- " + p.commonName + " (" + p.id + ")\n";
|
||||
}
|
||||
writeFileSync(rp, report, "utf-8");
|
||||
console.log("Report: " + rp);
|
||||
} else {
|
||||
console.log("\nALL PLANTS HAVE IMAGES!");
|
||||
}
|
||||
|
||||
closeDb();
|
||||
}
|
||||
|
||||
main().catch((err: any) => {
|
||||
console.error("Error:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
308
scripts/fill-plant-images.ts
Normal file
308
scripts/fill-plant-images.ts
Normal file
@@ -0,0 +1,308 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* fill-plant-images.ts — Fetch plant images from Wikipedia for plants missing them.
|
||||
*
|
||||
* Uses the Wikipedia API to search for the plant's scientific name
|
||||
* and grab the page thumbnail.
|
||||
*
|
||||
* Usage: cd apps/web && npx tsx scripts/fill-plant-images.ts
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
|
||||
// Load env
|
||||
const envPath = resolve(__dirname, "../.env.development");
|
||||
try {
|
||||
const env = readFileSync(envPath, "utf-8");
|
||||
for (const line of env.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed && !trimmed.startsWith("#")) {
|
||||
const eqIdx = trimmed.indexOf("=");
|
||||
if (eqIdx > 0) {
|
||||
const key = trimmed.slice(0, eqIdx).trim();
|
||||
const val = trimmed.slice(eqIdx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
import { getDb, closeDb } from "../src/lib/db/index";
|
||||
import { plants } from "../src/lib/db/schema";
|
||||
import { createClient } from "@libsql/client";
|
||||
import { sql } from "drizzle-orm";
|
||||
|
||||
const WIKI_API = "https://en.wikipedia.org/w/api.php";
|
||||
const UA = "PlantHealthKB/1.0 (plant-images)";
|
||||
const DELAY_MS = 500;
|
||||
const BATCH_SIZE = 50;
|
||||
|
||||
/** Direct page lookup by title — more reliable for known scientific names. */
|
||||
async function directPageLookup(title: string): Promise<string | null> {
|
||||
const params = new URLSearchParams({
|
||||
action: "query",
|
||||
titles: title,
|
||||
prop: "pageimages",
|
||||
pithumbsize: "400",
|
||||
format: "json",
|
||||
origin: "*",
|
||||
});
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
const res = await fetch(`${WIKI_API}?${params}`, {
|
||||
headers: { "User-Agent": UA },
|
||||
});
|
||||
if (res.status === 429) {
|
||||
await new Promise((r) => setTimeout(r, 3000 * 2 ** attempt));
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as {
|
||||
query?: { pages?: Record<string, { thumbnail?: { source: string }; missing?: boolean }> };
|
||||
};
|
||||
const pages = data?.query?.pages;
|
||||
if (!pages) return null;
|
||||
for (const [, p] of Object.entries(pages)) {
|
||||
if (!p.missing && p.thumbnail?.source) return p.thumbnail.source;
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("\n🌿 Fetching plant images from Wikipedia\n");
|
||||
|
||||
const db = getDb();
|
||||
const allPlants = await db
|
||||
.select({ id: plants.id, commonName: plants.commonName, scientificName: plants.scientificName })
|
||||
.from(plants)
|
||||
.where(sql`(image_url IS NULL OR image_url = '')`)
|
||||
.all();
|
||||
|
||||
console.log(`📋 ${allPlants.length} plants need images\n`);
|
||||
|
||||
if (allPlants.length === 0) {
|
||||
console.log("✅ All plants already have images!\n");
|
||||
closeDb();
|
||||
return;
|
||||
}
|
||||
|
||||
const rawClient = createClient({
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
});
|
||||
|
||||
let found = 0;
|
||||
const updates: { id: string; url: string }[] = [];
|
||||
|
||||
// Phase 1: Try direct page lookup by scientific name (most accurate)
|
||||
console.log("─── Phase 1: Direct page lookup ───\n");
|
||||
|
||||
for (let i = 0; i < allPlants.length; i++) {
|
||||
const plant = allPlants[i];
|
||||
const sciName = plant.scientificName
|
||||
.replace(/[×'"]/g, "")
|
||||
.replace(/\s*spp\.?\s*/i, "")
|
||||
.trim();
|
||||
|
||||
process.stdout.write(
|
||||
` [${String(i + 1).padStart(3)}/${allPlants.length}] ${plant.commonName.padEnd(30)} `,
|
||||
);
|
||||
|
||||
let url: string | null = null;
|
||||
|
||||
// Try scientific name first
|
||||
if (sciName && sciName !== "Unknown" && sciName !== "Various") {
|
||||
url = await directPageLookup(sciName);
|
||||
}
|
||||
|
||||
// Try common name if scientific name didn't work
|
||||
if (!url) {
|
||||
url = await directPageLookup(plant.commonName);
|
||||
}
|
||||
|
||||
// Try genus name
|
||||
if (!url && sciName) {
|
||||
const genus = sciName.split(/\s+/)[0];
|
||||
if (genus && genus.length > 3) {
|
||||
url = await directPageLookup(genus);
|
||||
}
|
||||
}
|
||||
|
||||
if (url) {
|
||||
updates.push({ id: plant.id, url });
|
||||
found++;
|
||||
process.stdout.write("✅\n");
|
||||
} else {
|
||||
process.stdout.write("⏭️\n");
|
||||
}
|
||||
|
||||
// Flush to DB in batches
|
||||
if (updates.length >= BATCH_SIZE) {
|
||||
await rawClient.batch(
|
||||
updates.map((u) => ({
|
||||
sql: "UPDATE plants SET image_url = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
args: [u.url, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
console.log(` → Flushed ${updates.length} to DB`);
|
||||
updates.length = 0;
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, DELAY_MS));
|
||||
}
|
||||
|
||||
// Flush remaining
|
||||
if (updates.length > 0) {
|
||||
await rawClient.batch(
|
||||
updates.map((u) => ({
|
||||
sql: "UPDATE plants SET image_url = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
args: [u.url, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
console.log(` → Flushed ${updates.length} to DB`);
|
||||
updates.length = 0;
|
||||
}
|
||||
|
||||
console.log(`\n✅ Phase 1 done: ${found}/${allPlants.length} plants got images\n`);
|
||||
|
||||
// Phase 2: Try remaining via search API
|
||||
const stillMissing = await db
|
||||
.select({ id: plants.id, commonName: plants.commonName, scientificName: plants.scientificName })
|
||||
.from(plants)
|
||||
.where(sql`(image_url IS NULL OR image_url = '')`)
|
||||
.all();
|
||||
|
||||
if (stillMissing.length > 0) {
|
||||
console.log(`─── Phase 2: Search API for ${stillMissing.length} remaining ───\n`);
|
||||
|
||||
for (let i = 0; i < stillMissing.length; i++) {
|
||||
const plant = stillMissing[i];
|
||||
const sciName = plant.scientificName.replace(/[×'"]/g, "").trim();
|
||||
|
||||
process.stdout.write(
|
||||
` [${String(i + 1).padStart(3)}/${stillMissing.length}] ${plant.commonName.padEnd(30)} `,
|
||||
);
|
||||
|
||||
// Search with scientific name
|
||||
const searchTerm = `${sciName} ${plant.commonName}`;
|
||||
const params = new URLSearchParams({
|
||||
action: "query",
|
||||
list: "search",
|
||||
srsearch: searchTerm,
|
||||
srlimit: "3",
|
||||
format: "json",
|
||||
origin: "*",
|
||||
});
|
||||
|
||||
let url: string | null = null;
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
const res = await fetch(`${WIKI_API}?${params}`, {
|
||||
headers: { "User-Agent": UA },
|
||||
});
|
||||
if (res.status === 429) {
|
||||
await new Promise((r) => setTimeout(r, 3000 * 2 ** attempt));
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) break;
|
||||
const data = (await res.json()) as {
|
||||
query?: { search?: Array<{ title: string; pageid: number }> };
|
||||
};
|
||||
const hits = data?.query?.search ?? [];
|
||||
if (hits.length === 0) break;
|
||||
|
||||
// Get thumbnail for first result
|
||||
for (const hit of hits) {
|
||||
const pageParams = new URLSearchParams({
|
||||
action: "query",
|
||||
pageids: String(hit.pageid),
|
||||
prop: "pageimages",
|
||||
pithumbsize: "400",
|
||||
format: "json",
|
||||
origin: "*",
|
||||
});
|
||||
const pageRes = await fetch(`${WIKI_API}?${pageParams}`, {
|
||||
headers: { "User-Agent": UA },
|
||||
});
|
||||
if (!pageRes.ok) continue;
|
||||
const pageData = (await pageRes.json()) as {
|
||||
query?: { pages?: Record<string, { thumbnail?: { source: string } }> };
|
||||
};
|
||||
const pages = pageData?.query?.pages;
|
||||
if (!pages) continue;
|
||||
for (const [, p] of Object.entries(pages)) {
|
||||
if (p.thumbnail?.source) {
|
||||
url = p.thumbnail.source;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (url) break;
|
||||
}
|
||||
break;
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
}
|
||||
|
||||
if (url) {
|
||||
await rawClient.execute({
|
||||
sql: "UPDATE plants SET image_url = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
args: [url, plant.id],
|
||||
});
|
||||
found++;
|
||||
process.stdout.write("✅\n");
|
||||
} else {
|
||||
process.stdout.write("❌\n");
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, DELAY_MS));
|
||||
}
|
||||
}
|
||||
|
||||
// Final count
|
||||
const final = await db
|
||||
.select({ id: plants.id, commonName: plants.commonName, imageUrl: plants.imageUrl })
|
||||
.from(plants)
|
||||
.all();
|
||||
const withImg = final.filter((p) => p.imageUrl);
|
||||
const withoutImg = final.filter((p) => !p.imageUrl);
|
||||
|
||||
console.log(`\n${"═".repeat(50)}`);
|
||||
console.log(`📊 FINAL: ${final.length} plants`);
|
||||
console.log(` With images: ${withImg.length}`);
|
||||
console.log(` Missing images: ${withoutImg.length}`);
|
||||
|
||||
if (withoutImg.length > 0) {
|
||||
console.log(`\n📝 Plants still needing images:`);
|
||||
withoutImg.forEach((p) => console.log(` ❌ ${p.id}: ${p.commonName}`));
|
||||
// Save to file for reference
|
||||
const reportPath = resolve(__dirname, ".plant-image-review-needed.md");
|
||||
let report = "# Plant Images — Still Missing\n\n";
|
||||
report += `Generated: ${new Date().toISOString()}\n\n`;
|
||||
report += `## 🚫 Plants without images (${withoutImg.length})\n\n`;
|
||||
for (const p of withoutImg) {
|
||||
report += `- **${p.commonName}** (\`${p.id}\`)\n`;
|
||||
}
|
||||
writeFileSync(reportPath, report, "utf-8");
|
||||
console.log(` 📝 Review report: ${reportPath}`);
|
||||
} else {
|
||||
console.log("\n✅ All plants now have images!");
|
||||
}
|
||||
|
||||
rawClient.close();
|
||||
closeDb();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\n❌", err);
|
||||
process.exit(1);
|
||||
});
|
||||
927
scripts/fill-training-dataset.ts
Normal file
927
scripts/fill-training-dataset.ts
Normal file
@@ -0,0 +1,927 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* fill-training-dataset.ts
|
||||
*
|
||||
* Scans the existing dataset directory and downloads any missing images
|
||||
* to reach the target counts (200 per disease, 400 for healthy).
|
||||
*
|
||||
* Does NOT re-run prevalence queries — just fills gaps from image sources.
|
||||
* Each run scans the directory, reports deficits, then fills them.
|
||||
* Interrupt-safe: re-run to pick up where you left off.
|
||||
*
|
||||
* Parallelism strategy:
|
||||
* - Disease-level: 30 diseases processed concurrently
|
||||
* - Per disease: all 3 DDG queries run in parallel
|
||||
* - Per query: all search pages fetched in parallel
|
||||
* - Per disease: DDG, iNaturalist, and Wikimedia Commons all run concurrently
|
||||
* - A shared DDG token-bucket rate limiter prevents bans
|
||||
*
|
||||
* Usage: cd apps/web && npx tsx scripts/fill-training-dataset.ts
|
||||
*/
|
||||
|
||||
import "dotenv/config";
|
||||
import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from "fs";
|
||||
import { resolve, extname } from "path";
|
||||
|
||||
// Load .env.development for DB creds
|
||||
const envPath = resolve(__dirname, "../.env.development");
|
||||
try {
|
||||
const env = readFileSync(envPath, "utf-8");
|
||||
for (const line of env.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed && !trimmed.startsWith("#")) {
|
||||
const eqIdx = trimmed.indexOf("=");
|
||||
if (eqIdx > 0) {
|
||||
const key = trimmed.slice(0, eqIdx).trim();
|
||||
const val = trimmed.slice(eqIdx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
import { getDb, closeDb } from "@/lib/db/index";
|
||||
import { diseases } from "@/lib/db/schema";
|
||||
|
||||
// ─── Config ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const DATASET_DIR = resolve(__dirname, "../data/dataset");
|
||||
const SEEN_CACHE_FILE = resolve(DATASET_DIR, ".fill-seen-urls.json");
|
||||
|
||||
/** Target images per disease */
|
||||
const TARGET_PER_DISEASE = 200;
|
||||
|
||||
/** Target images for the "healthy" class */
|
||||
const TARGET_HEALTHY = 400;
|
||||
|
||||
/**
|
||||
* How many diseases to process in parallel.
|
||||
* Each disease is I/O-bound (HTTP requests), so high concurrency is safe.
|
||||
* The global DDG rate limiter prevents us from overwhelming DuckDuckGo.
|
||||
*/
|
||||
const DISEASE_CONCURRENCY = 20;
|
||||
|
||||
/**
|
||||
* Max DDG requests per second (shared across all concurrent diseases).
|
||||
* DuckDuckGo is fairly tolerant, but we still want to be polite.
|
||||
* With DISEASE_CONCURRENCY=30, each disease fires 3 parallel queries with
|
||||
* parallel pages = 9 parallel DDG requests per disease at peak.
|
||||
* The rate limiter serializes this so we don't get banned.
|
||||
*/
|
||||
const DDG_RATE_LIMIT_RPS = 2;
|
||||
|
||||
/** Max concurrent image downloads per disease */
|
||||
const CONCURRENT_DOWNLOADS = 2;
|
||||
|
||||
/** Minimum image size in bytes to accept */
|
||||
const MIN_IMAGE_SIZE = 10_000; // 10KB
|
||||
|
||||
/** Maximum image size in bytes */
|
||||
const MAX_IMAGE_SIZE = 10 * 1024 * 1024; // 10MB
|
||||
|
||||
/** Allowed file extensions */
|
||||
const ALLOWED_EXTENSIONS = [".jpg", ".jpeg", ".png", ".webp"];
|
||||
|
||||
/** User agent for requests */
|
||||
const UA =
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1";
|
||||
|
||||
/** Healthy class directory name */
|
||||
const HEALTHY_CLASS = "healthy";
|
||||
|
||||
/** How often (in diseases processed) to flush the seen-URLs cache to disk */
|
||||
const SEEN_CACHE_FLUSH_INTERVAL = 20;
|
||||
|
||||
/** Max DDG pages to fetch per query.
|
||||
* Each page returns ~100 image results, so 3 pages × 3 queries = ~900 raw URLs
|
||||
* before dedup — more than enough to find 200 unique, valid images. */
|
||||
const MAX_DDG_PAGES = 3;
|
||||
|
||||
/** Healthy source queries limit */
|
||||
const MAX_HEALTHY_QUERIES = 20;
|
||||
|
||||
// ─── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
interface DuckDuckGoImageResult {
|
||||
image: string;
|
||||
title: string;
|
||||
url: string;
|
||||
thumbnail: string;
|
||||
height: number;
|
||||
width: number;
|
||||
}
|
||||
|
||||
interface DiseaseInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
plantId: string;
|
||||
have: number;
|
||||
needed: number;
|
||||
}
|
||||
|
||||
interface CollectResult {
|
||||
urls: string[];
|
||||
exhausted: boolean;
|
||||
}
|
||||
|
||||
// ─── Token-Bucket Rate Limiter ──────────────────────────────────────────────
|
||||
|
||||
class TokenBucket {
|
||||
private tokens: number;
|
||||
private lastRefill: number;
|
||||
private readonly capacity: number;
|
||||
private readonly refillInterval: number; // ms per token (e.g., 100ms for 10 rps)
|
||||
|
||||
constructor(rps: number) {
|
||||
this.capacity = rps;
|
||||
this.tokens = rps;
|
||||
this.lastRefill = Date.now();
|
||||
this.refillInterval = 1000 / rps;
|
||||
}
|
||||
|
||||
/** Acquire one token, blocking until one is available. */
|
||||
async acquire(): Promise<void> {
|
||||
while (true) {
|
||||
this.refill();
|
||||
if (this.tokens >= 1) {
|
||||
this.tokens -= 1;
|
||||
return;
|
||||
}
|
||||
// No tokens — wait for the next one to arrive, then retry
|
||||
await sleep(Math.ceil(this.refillInterval));
|
||||
}
|
||||
}
|
||||
|
||||
private refill(): void {
|
||||
const now = Date.now();
|
||||
const elapsed = now - this.lastRefill;
|
||||
const newTokens = Math.floor(elapsed / this.refillInterval);
|
||||
if (newTokens > 0) {
|
||||
this.tokens = Math.min(this.capacity, this.tokens + newTokens);
|
||||
this.lastRefill = now - (elapsed % this.refillInterval);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Global DDG rate limiter — all concurrent diseases share this
|
||||
const ddgLimiter = new TokenBucket(DDG_RATE_LIMIT_RPS);
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Count actual image files in a directory (matching img_* pattern). */
|
||||
function countImagesInDir(dir: string): number {
|
||||
if (!existsSync(dir)) return 0;
|
||||
try {
|
||||
const files = readdirSync(dir);
|
||||
return files.filter((f) => f.startsWith("img_")).length;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Seen-URLs Cache ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Load the per-disease seen-URLs cache from disk.
|
||||
* This prevents re-fetching the same URLs across runs.
|
||||
*/
|
||||
function loadSeenUrlsCache(): Record<string, string[]> {
|
||||
if (existsSync(SEEN_CACHE_FILE)) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(SEEN_CACHE_FILE, "utf-8"));
|
||||
} catch {}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the seen-URLs cache to disk.
|
||||
*/
|
||||
function saveSeenUrlsCache(cache: Record<string, string[]>): void {
|
||||
writeFileSync(SEEN_CACHE_FILE, JSON.stringify(cache, null, 2));
|
||||
}
|
||||
|
||||
// ─── DDG VQD Token Cache ──────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Simple in-memory cache for DDG VQD tokens.
|
||||
* Tokens are per-query, but if we've fetched one for a similar query recently,
|
||||
* we can skip the initial HTML page fetch.
|
||||
*/
|
||||
const vqdCache = new Map<string, { token: string; expiresAt: number }>();
|
||||
|
||||
function getCachedVqd(query: string): string | undefined {
|
||||
const entry = vqdCache.get(query);
|
||||
if (entry && entry.expiresAt > Date.now()) return entry.token;
|
||||
vqdCache.delete(query);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function setCachedVqd(query: string, token: string): void {
|
||||
// VQD tokens seem to be valid for a few minutes; cache for 5 min
|
||||
vqdCache.set(query, { token, expiresAt: Date.now() + 5 * 60 * 1000 });
|
||||
// Evict oldest entries if cache grows too large (unlikely but safe)
|
||||
if (vqdCache.size > 500) {
|
||||
const firstKey = vqdCache.keys().next().value;
|
||||
if (firstKey) vqdCache.delete(firstKey);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── DuckDuckGo API ─────────────────────────────────────────────────────────
|
||||
|
||||
async function getVqdToken(query: string): Promise<string> {
|
||||
const cached = getCachedVqd(query);
|
||||
if (cached) return cached;
|
||||
|
||||
const url = `https://duckduckgo.com/?q=${encodeURIComponent(query)}&t=h_&iax=images&ia=images`;
|
||||
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": UA, Accept: "text/html" },
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
|
||||
if (!res.ok) throw new Error(`Failed to get vqd token: ${res.status}`);
|
||||
|
||||
const html = await res.text();
|
||||
const match = html.match(/vqd['"]?\s*[:=]\s*['"]([a-f0-9-]+)['"]/);
|
||||
if (!match) throw new Error(`Could not extract vqd token for "${query}"`);
|
||||
|
||||
setCachedVqd(query, match[1]);
|
||||
return match[1];
|
||||
}
|
||||
|
||||
async function searchImagesDuckDuckGo(
|
||||
query: string,
|
||||
vqd: string,
|
||||
page: number,
|
||||
): Promise<DuckDuckGoImageResult[]> {
|
||||
// Rate-limit before making the request
|
||||
await ddgLimiter.acquire();
|
||||
|
||||
const url = `https://duckduckgo.com/i.js?q=${encodeURIComponent(
|
||||
query,
|
||||
)}&vqd=${vqd}&o=json&p=${page}&f=,,,`;
|
||||
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": UA,
|
||||
Accept: "application/json",
|
||||
Referer: `https://duckduckgo.com/?q=${encodeURIComponent(query)}&t=h_&iax=images&ia=images`,
|
||||
},
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
if (res.status === 429) {
|
||||
// Rate limited — wait and retry once
|
||||
await sleep(5_000);
|
||||
return searchImagesDuckDuckGo(query, vqd, page);
|
||||
}
|
||||
if (res.status === 403) return [];
|
||||
// Don't throw for transient errors — just return empty
|
||||
return [];
|
||||
}
|
||||
|
||||
const data = (await res.json()) as { results: DuckDuckGoImageResult[] };
|
||||
return data.results ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect images from DDG for a single query.
|
||||
* Fetches up to MAX_DDG_PAGES pages in PARALLEL (rate-limited via ddgLimiter).
|
||||
*/
|
||||
async function collectFromDdgQuery(
|
||||
query: string,
|
||||
target: number,
|
||||
seenUrls: Set<string>,
|
||||
): Promise<CollectResult> {
|
||||
const results: string[] = [];
|
||||
|
||||
let vqd: string;
|
||||
try {
|
||||
vqd = await getVqdToken(query);
|
||||
} catch (err) {
|
||||
console.warn(` ⚠ DDG token failed: ${err instanceof Error ? err.message : "unknown"}`);
|
||||
return { urls: [], exhausted: true };
|
||||
}
|
||||
|
||||
// Fetch all pages in parallel
|
||||
const pageFetches: Promise<DuckDuckGoImageResult[]>[] = [];
|
||||
for (let page = 1; page <= MAX_DDG_PAGES; page++) {
|
||||
pageFetches.push(searchImagesDuckDuckGo(query, vqd, page));
|
||||
}
|
||||
|
||||
const pageResults = await Promise.allSettled(pageFetches);
|
||||
|
||||
for (const settled of pageResults) {
|
||||
if (settled.status !== "fulfilled") continue;
|
||||
if (results.length >= target) break;
|
||||
|
||||
for (const r of settled.value) {
|
||||
if (results.length >= target) break;
|
||||
const imgUrl = r.image || r.url;
|
||||
if (!imgUrl || typeof imgUrl !== "string") continue;
|
||||
if (seenUrls.has(imgUrl)) continue;
|
||||
try {
|
||||
new URL(imgUrl);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
seenUrls.add(imgUrl);
|
||||
results.push(imgUrl);
|
||||
}
|
||||
}
|
||||
|
||||
return { urls: results.slice(0, target), exhausted: results.length < target };
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect images from DDG across ALL queries for a disease.
|
||||
* Runs all queries in PARALLEL, then merges deduplicated results.
|
||||
*/
|
||||
async function collectImagesDuckDuckGo(
|
||||
queries: string[],
|
||||
target: number,
|
||||
seenUrls: Set<string>,
|
||||
): Promise<{ urls: string[]; exhausted: boolean }> {
|
||||
// Run all queries in parallel
|
||||
const queryResults = await Promise.allSettled(
|
||||
queries.map((q) => collectFromDdgQuery(q, target, seenUrls)),
|
||||
);
|
||||
|
||||
// Merge results — seenUrls already deduplicates across queries
|
||||
const merged: string[] = [];
|
||||
for (const settled of queryResults) {
|
||||
if (settled.status === "fulfilled") {
|
||||
merged.push(...settled.value.urls);
|
||||
if (merged.length >= target) break;
|
||||
}
|
||||
}
|
||||
|
||||
return { urls: merged.slice(0, target), exhausted: merged.length < target };
|
||||
}
|
||||
|
||||
// ─── iNaturalist API ───────────────────────────────────────────────────────
|
||||
|
||||
async function searchImagesInaturalist(
|
||||
query: string,
|
||||
target: number,
|
||||
seenUrls: Set<string>,
|
||||
): Promise<CollectResult> {
|
||||
const results: string[] = [];
|
||||
const perPage = Math.min(target, 200);
|
||||
|
||||
const apiUrl =
|
||||
`https://api.inaturalist.org/v1/observations` +
|
||||
`?q=${encodeURIComponent(query)}` +
|
||||
`&photos_only=true` +
|
||||
`&quality_grade=research` +
|
||||
`&per_page=${perPage}` +
|
||||
`&order_by=observed_on&order=desc`;
|
||||
|
||||
try {
|
||||
const res = await fetch(apiUrl, {
|
||||
headers: { "User-Agent": UA, Accept: "application/json" },
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
if (!res.ok) return { urls: [], exhausted: false };
|
||||
|
||||
const data = (await res.json()) as {
|
||||
results: Array<{ photos: Array<{ url: string }> }>;
|
||||
};
|
||||
|
||||
for (const obs of data.results ?? []) {
|
||||
if (results.length >= target) break;
|
||||
for (const photo of obs.photos ?? []) {
|
||||
if (results.length >= target) break;
|
||||
const url = photo.url;
|
||||
if (!url || seenUrls.has(url)) continue;
|
||||
const fullUrl = url.replace("/medium.", "/original.");
|
||||
seenUrls.add(fullUrl);
|
||||
results.push(fullUrl);
|
||||
}
|
||||
}
|
||||
|
||||
return { urls: results, exhausted: results.length < target };
|
||||
} catch {
|
||||
return { urls: results, exhausted: false };
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Wikimedia Commons API ─────────────────────────────────────────────────
|
||||
|
||||
async function searchImagesCommons(
|
||||
query: string,
|
||||
target: number,
|
||||
seenUrls: Set<string>,
|
||||
): Promise<CollectResult> {
|
||||
const results: string[] = [];
|
||||
let sroffset = 0;
|
||||
|
||||
while (results.length < target) {
|
||||
const params = new URLSearchParams({
|
||||
action: "query",
|
||||
list: "search",
|
||||
srsearch: query,
|
||||
srnamespace: "6",
|
||||
srlimit: "50",
|
||||
sroffset: String(sroffset),
|
||||
format: "json",
|
||||
});
|
||||
|
||||
const url = `https://commons.wikimedia.org/w/api.php?${params}`;
|
||||
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": UA },
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!res.ok) break;
|
||||
|
||||
const data = (await res.json()) as {
|
||||
query?: { search?: Array<{ title: string }> };
|
||||
continue?: { sroffset?: number };
|
||||
};
|
||||
|
||||
const hits = data.query?.search ?? [];
|
||||
if (hits.length === 0) break;
|
||||
|
||||
for (const hit of hits) {
|
||||
if (results.length >= target) break;
|
||||
const filename = hit.title.replace(/^File:/, "");
|
||||
const imgUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(
|
||||
filename,
|
||||
)}`;
|
||||
if (seenUrls.has(imgUrl)) continue;
|
||||
seenUrls.add(imgUrl);
|
||||
results.push(imgUrl);
|
||||
}
|
||||
|
||||
sroffset = data.continue?.sroffset ?? sroffset + hits.length;
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { urls: results, exhausted: results.length < target };
|
||||
}
|
||||
|
||||
// ─── Image Download ─────────────────────────────────────────────────────────
|
||||
|
||||
async function downloadImage(url: string, destPath: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: { "User-Agent": UA, Accept: "image/webp,image/png,image/jpeg,*/*" },
|
||||
signal: AbortSignal.timeout(8_000),
|
||||
});
|
||||
if (!res.ok) return false;
|
||||
|
||||
const contentType = res.headers.get("content-type") || "";
|
||||
if (contentType.includes("text/html")) return false;
|
||||
|
||||
const buffer = Buffer.from(await res.arrayBuffer());
|
||||
if (buffer.length < MIN_IMAGE_SIZE) return false;
|
||||
if (buffer.length > MAX_IMAGE_SIZE) return false;
|
||||
|
||||
let ext = extname(new URL(url).pathname).toLowerCase();
|
||||
if (!ALLOWED_EXTENSIONS.includes(ext)) {
|
||||
if (contentType.includes("jpeg") || contentType.includes("jpg")) ext = ".jpg";
|
||||
else if (contentType.includes("png")) ext = ".png";
|
||||
else if (contentType.includes("webp")) ext = ".webp";
|
||||
else ext = ".jpg";
|
||||
}
|
||||
|
||||
const filePath = destPath.replace(/\.\w+$/, ext);
|
||||
writeFileSync(filePath, buffer);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadBatch(
|
||||
urls: string[],
|
||||
classDir: string,
|
||||
startIndex: number,
|
||||
): Promise<{ downloaded: number; failed: number; lastIndex: number }> {
|
||||
let downloaded = 0;
|
||||
let failed = 0;
|
||||
let index = startIndex;
|
||||
|
||||
for (let i = 0; i < urls.length; i += CONCURRENT_DOWNLOADS) {
|
||||
const chunk = urls.slice(i, i + CONCURRENT_DOWNLOADS);
|
||||
|
||||
const results = await Promise.all(
|
||||
chunk.map(async (url) => {
|
||||
const paddedIndex = String(index).padStart(4, "0");
|
||||
const destPath = resolve(classDir, `img_${paddedIndex}.jpg`);
|
||||
const success = await downloadImage(url, destPath);
|
||||
return { success, index: index++ };
|
||||
}),
|
||||
);
|
||||
|
||||
for (const r of results) {
|
||||
if (r.success) downloaded++;
|
||||
else failed++;
|
||||
}
|
||||
}
|
||||
|
||||
return { downloaded, failed, lastIndex: index };
|
||||
}
|
||||
|
||||
// ─── Query Building ─────────────────────────────────────────────────────────
|
||||
|
||||
function buildSearchQueries(name: string, plant: string): string[] {
|
||||
return [`${name} ${plant} leaf disease`, `${plant} ${name} symptoms`, `${name} ${plant}`];
|
||||
}
|
||||
|
||||
function buildHealthyQueries(plant: string): string[] {
|
||||
const name = plant.replace(/-/g, " ");
|
||||
return [
|
||||
`healthy ${name} leaf`,
|
||||
`${name} leaf closeup`,
|
||||
`healthy ${name} plant`,
|
||||
`${name} foliage`,
|
||||
];
|
||||
}
|
||||
|
||||
// ─── Fill Logic ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Try to collect up to `needed` images for a disease by hitting all three
|
||||
* sources IN PARALLEL. Returns how many new images were actually downloaded.
|
||||
*
|
||||
* Sources (DDG with its 3 internal queries, iNat, Commons) all run concurrently.
|
||||
* As soon as any source completes, its URLs are downloaded immediately while
|
||||
* other sources are still searching (pipeline).
|
||||
*/
|
||||
async function fillClass(
|
||||
_diseaseId: string,
|
||||
queries: string[],
|
||||
needed: number,
|
||||
classDir: string,
|
||||
seenUrls: Set<string>,
|
||||
): Promise<number> {
|
||||
if (needed <= 0) return 0;
|
||||
|
||||
mkdirSync(classDir, { recursive: true });
|
||||
const startCount = countImagesInDir(classDir);
|
||||
|
||||
// ── Run all sources in parallel, pipelining downloads ──────────────────
|
||||
// Start downloading from each source as soon as it returns results, rather
|
||||
// than waiting for all sources to complete. DDG is (by far) the richest
|
||||
// source, so its results start saving to disk while iNat and Commons are
|
||||
// still searching.
|
||||
//
|
||||
// Each source gets a DEDICATED index range so there's no race condition
|
||||
// writing files. DDG gets [startCount, startCount+199], iNat gets
|
||||
// [startCount+200, startCount+399], Commons gets [startCount+400,...].
|
||||
// The 4-digit filename supports up to 9999, well beyond our 200 target.
|
||||
|
||||
let totalDownloaded = 0;
|
||||
let totalFailed = 0;
|
||||
let anySuccess = false;
|
||||
|
||||
const collectAndDownload = async (
|
||||
label: string,
|
||||
collector: () => Promise<CollectResult>,
|
||||
indexOffset: number,
|
||||
): Promise<void> => {
|
||||
const result = await collector();
|
||||
if (result.urls.length === 0) return;
|
||||
console.log(` ${label}: ${result.urls.length} new URLs`);
|
||||
|
||||
// Each source writes to its own non-overlapping range
|
||||
const { downloaded, failed } = await downloadBatch(result.urls, classDir, indexOffset);
|
||||
totalDownloaded += downloaded;
|
||||
totalFailed += failed;
|
||||
if (downloaded > 0) anySuccess = true;
|
||||
};
|
||||
|
||||
await Promise.allSettled([
|
||||
collectAndDownload("DDG", () => collectImagesDuckDuckGo(queries, needed, seenUrls), startCount),
|
||||
collectAndDownload(
|
||||
"iNat",
|
||||
() => searchImagesInaturalist(queries[0], needed, seenUrls),
|
||||
startCount + TARGET_PER_DISEASE,
|
||||
),
|
||||
collectAndDownload(
|
||||
"Commons",
|
||||
() => searchImagesCommons(queries[0], needed, seenUrls),
|
||||
startCount + 2 * TARGET_PER_DISEASE,
|
||||
),
|
||||
]);
|
||||
|
||||
if (!anySuccess) {
|
||||
console.log(` ✗ No new images found from any source`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const newTotal = countImagesInDir(classDir);
|
||||
const gained = newTotal - startCount;
|
||||
console.log(
|
||||
` ✓ ${totalDownloaded}/${totalDownloaded + totalFailed} downloaded` +
|
||||
` (${totalFailed} failed, ${gained} new files)`,
|
||||
);
|
||||
|
||||
return gained;
|
||||
}
|
||||
|
||||
// ─── Directory Scanner ─────────────────────────────────────────────────────
|
||||
|
||||
interface ScanResult {
|
||||
/** Disease id → how many images currently on disk */
|
||||
diseaseCounts: Map<string, number>;
|
||||
/** How many healthy images on disk */
|
||||
healthyCount: number;
|
||||
}
|
||||
|
||||
function scanDataset(): ScanResult {
|
||||
const diseaseCounts = new Map<string, number>();
|
||||
let healthyCount = 0;
|
||||
|
||||
if (!existsSync(DATASET_DIR)) {
|
||||
return { diseaseCounts, healthyCount: 0 };
|
||||
}
|
||||
|
||||
const entries = readdirSync(DATASET_DIR, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (entry.name.startsWith(".")) continue;
|
||||
|
||||
if (entry.name === HEALTHY_CLASS) {
|
||||
healthyCount = countImagesInDir(resolve(DATASET_DIR, entry.name));
|
||||
} else {
|
||||
const count = countImagesInDir(resolve(DATASET_DIR, entry.name));
|
||||
if (count > 0) {
|
||||
diseaseCounts.set(entry.name, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { diseaseCounts, healthyCount };
|
||||
}
|
||||
|
||||
// ─── CLI Flags ──────────────────────────────────────────────────────────────
|
||||
|
||||
function parseFlags(): { reverse: boolean } {
|
||||
const args = process.argv.slice(2);
|
||||
return {
|
||||
reverse: args.includes("--reverse") || args.includes("-r"),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const flags = parseFlags();
|
||||
|
||||
console.log("=".repeat(60));
|
||||
console.log("TRAINING DATASET FILL — Parallelized gap-filling download");
|
||||
if (flags.reverse) console.log(" (reverse order — processing from lowest deficit first)");
|
||||
console.log("=".repeat(60));
|
||||
|
||||
// Ensure dataset directory exists
|
||||
mkdirSync(DATASET_DIR, { recursive: true });
|
||||
|
||||
// ── Step 1: Scan what we already have ────────────────────────────────────
|
||||
console.log("\nScanning existing dataset...");
|
||||
const { diseaseCounts, healthyCount } = scanDataset();
|
||||
console.log(` Found ${diseaseCounts.size} disease directories, ${healthyCount} healthy images`);
|
||||
|
||||
// ── Step 2: Load disease info from DB ────────────────────────────────────
|
||||
console.log("\nLoading disease info from database...");
|
||||
const db = getDb();
|
||||
|
||||
const allDiseases = await db
|
||||
.select({
|
||||
id: diseases.id,
|
||||
plantId: diseases.plantId,
|
||||
name: diseases.name,
|
||||
})
|
||||
.from(diseases);
|
||||
|
||||
// Build a deduplicated map: disease id → first disease info found
|
||||
const diseaseInfo = new Map<string, { name: string; plantId: string }>();
|
||||
for (const d of allDiseases) {
|
||||
if (!diseaseInfo.has(d.id)) {
|
||||
diseaseInfo.set(d.id, { name: d.name, plantId: d.plantId });
|
||||
}
|
||||
}
|
||||
console.log(` Loaded ${diseaseInfo.size} unique diseases from DB`);
|
||||
|
||||
// ── Step 3: Build deficit list ──────────────────────────────────────────
|
||||
const deficits: DiseaseInfo[] = [];
|
||||
|
||||
for (const [id, info] of diseaseInfo) {
|
||||
const have = diseaseCounts.get(id) ?? 0;
|
||||
const needed = TARGET_PER_DISEASE - have;
|
||||
if (needed > 0) {
|
||||
deficits.push({ id, name: info.name, plantId: info.plantId, have, needed });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by deficit size (largest first) so we prioritize the neediest diseases
|
||||
deficits.sort((a, b) => b.needed - a.needed);
|
||||
|
||||
// Reverse order if --reverse/-r flag is set (useful to try a different
|
||||
// direction when the front of the queue keeps hitting dead URLs)
|
||||
if (flags.reverse) deficits.reverse();
|
||||
|
||||
const healthyDeficit = TARGET_HEALTHY - healthyCount;
|
||||
|
||||
console.log(`\n${"=".repeat(60)}`);
|
||||
console.log("DEFICIT REPORT");
|
||||
console.log(`${"=".repeat(60)}`);
|
||||
console.log(` Diseases needing images: ${deficits.length}/${diseaseInfo.size}`);
|
||||
console.log(` Total images missing: ${deficits.reduce((s, d) => s + d.needed, 0)}`);
|
||||
console.log(` Healthy deficit: ${Math.max(0, healthyDeficit)}`);
|
||||
console.log(` Parallelism: ${DISEASE_CONCURRENCY} diseases at once`);
|
||||
console.log(` DDG rate limit: ${DDG_RATE_LIMIT_RPS} req/s (shared)`);
|
||||
console.log(
|
||||
` Order: ${flags.reverse ? "reverse (--reverse)" : "normal (deficit-first)"}`,
|
||||
);
|
||||
console.log(`${"=".repeat(60)}`);
|
||||
|
||||
if (deficits.length === 0 && healthyDeficit <= 0) {
|
||||
console.log("\n ✓ Nothing to do — all targets met!\n");
|
||||
await closeDb();
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Step 4: Load seen-URLs cache ────────────────────────────────────────
|
||||
const seenUrlsCache = loadSeenUrlsCache();
|
||||
let totalDownloaded = 0;
|
||||
let totalFailed = 0;
|
||||
let diseasesProcessed = 0;
|
||||
const startTime = Date.now();
|
||||
|
||||
// ── Step 5: Fill disease deficits ───────────────────────────────────────
|
||||
if (deficits.length > 0) {
|
||||
console.log("\n" + "─".repeat(60));
|
||||
console.log(`FILLING ${deficits.length} DISEASES (target: ${TARGET_PER_DISEASE} each)`);
|
||||
console.log("─".repeat(60));
|
||||
|
||||
// Process in parallel batches
|
||||
for (let i = 0; i < deficits.length; i += DISEASE_CONCURRENCY) {
|
||||
const batch = deficits.slice(i, i + DISEASE_CONCURRENCY);
|
||||
const batchNum = Math.floor(i / DISEASE_CONCURRENCY) + 1;
|
||||
const totalBatches = Math.ceil(deficits.length / DISEASE_CONCURRENCY);
|
||||
|
||||
console.log(`\n[Batch ${batchNum}/${totalBatches}] Processing ${batch.length} diseases...`);
|
||||
|
||||
// Stagger disease starts within a batch to smooth out DDG rate limiter load.
|
||||
// Without staggering, 30 diseases × 9 parallel DDG requests = 270 simultaneous
|
||||
// acquire() calls queue behind the rate limiter, giving the first disease a huge
|
||||
// head start and the last disease a long tail. Staggering by 200ms each spreads
|
||||
// the load evenly, reducing tail latency and improving overall throughput.
|
||||
const STAGGER_MS = 200;
|
||||
const batchResults = await Promise.allSettled(
|
||||
batch.map((d, idx) =>
|
||||
(async () => {
|
||||
if (idx > 0) await sleep(idx * STAGGER_MS);
|
||||
|
||||
const classDir = resolve(DATASET_DIR, d.id);
|
||||
const queries = buildSearchQueries(d.name, d.plantId);
|
||||
const seen = new Set<string>(seenUrlsCache[d.id] ?? []);
|
||||
|
||||
console.log(
|
||||
` [${d.id}] have ${d.have}, need ${d.needed} more` + ` (${d.name} / ${d.plantId})`,
|
||||
);
|
||||
|
||||
const gained = await fillClass(d.id, queries, d.needed, classDir, seen);
|
||||
|
||||
// Update seen-URLs cache for this disease
|
||||
seenUrlsCache[d.id] = Array.from(seen);
|
||||
return gained;
|
||||
})(),
|
||||
),
|
||||
);
|
||||
|
||||
// Aggregate batch results
|
||||
for (const result of batchResults) {
|
||||
if (result.status === "fulfilled") {
|
||||
totalDownloaded += result.value;
|
||||
} else {
|
||||
console.error(` ✗ Disease failed: ${result.reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
diseasesProcessed += batch.length;
|
||||
|
||||
// Flush seen-URLs cache to disk periodically (not after every disease)
|
||||
if (
|
||||
diseasesProcessed % SEEN_CACHE_FLUSH_INTERVAL < batch.length ||
|
||||
i + batch.length >= deficits.length
|
||||
) {
|
||||
saveSeenUrlsCache(seenUrlsCache);
|
||||
}
|
||||
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
const rate = diseasesProcessed / Math.max(1, elapsed);
|
||||
const remaining = deficits.length - diseasesProcessed;
|
||||
const eta = remaining / Math.max(0.01, rate);
|
||||
console.log(
|
||||
` [Batch ${batchNum}/${totalBatches}] checkpoint — ` +
|
||||
`${totalDownloaded} downloaded, ` +
|
||||
`${diseasesProcessed}/${deficits.length} diseases (${rate.toFixed(1)}/s, ` +
|
||||
`ETA: ${Math.round(eta)}s)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Step 6: Fill healthy deficit ────────────────────────────────────────
|
||||
if (healthyDeficit > 0) {
|
||||
console.log("\n" + "─".repeat(60));
|
||||
console.log(`FILLING HEALTHY CLASS (target: ${TARGET_HEALTHY})`);
|
||||
console.log("─".repeat(60));
|
||||
|
||||
const healthyDir = resolve(DATASET_DIR, HEALTHY_CLASS);
|
||||
mkdirSync(healthyDir, { recursive: true });
|
||||
|
||||
// Collect all unique plants from the disease info
|
||||
const allPlants = [...new Set(diseaseInfo.values())].map((d) => d.plantId);
|
||||
const allHealthyQueries: string[] = [];
|
||||
for (const plant of allPlants) {
|
||||
allHealthyQueries.push(...buildHealthyQueries(plant));
|
||||
}
|
||||
|
||||
const healthySeen = new Set<string>(seenUrlsCache[HEALTHY_CLASS] ?? []);
|
||||
const healthyNeeded = TARGET_HEALTHY - countImagesInDir(healthyDir);
|
||||
|
||||
// Run all 3 sources in parallel for the healthy class too
|
||||
const [ddgUrls, inatUrls, commonsUrls] = await Promise.allSettled([
|
||||
collectImagesDuckDuckGo(
|
||||
allHealthyQueries.slice(0, MAX_HEALTHY_QUERIES),
|
||||
healthyNeeded,
|
||||
healthySeen,
|
||||
),
|
||||
searchImagesInaturalist(allHealthyQueries[0], healthyNeeded, healthySeen),
|
||||
searchImagesCommons(allHealthyQueries[0], healthyNeeded, healthySeen),
|
||||
]);
|
||||
|
||||
const allUrls: string[] = [];
|
||||
for (const settled of [ddgUrls, inatUrls, commonsUrls]) {
|
||||
if (settled.status === "fulfilled") {
|
||||
allUrls.push(...settled.value.urls);
|
||||
}
|
||||
}
|
||||
|
||||
if (allUrls.length > 0) {
|
||||
console.log(`\n Downloading ${allUrls.length} healthy images...`);
|
||||
const startIdx = countImagesInDir(healthyDir);
|
||||
const { downloaded, failed } = await downloadBatch(allUrls, healthyDir, startIdx);
|
||||
|
||||
const newTotal = countImagesInDir(healthyDir);
|
||||
const gained = newTotal - healthyCount;
|
||||
totalDownloaded += gained;
|
||||
totalFailed += failed;
|
||||
|
||||
console.log(
|
||||
` ${downloaded > 0 ? "✓" : "✗"} Got ${downloaded} images.` +
|
||||
` Total healthy: ${newTotal}/${TARGET_HEALTHY} (${gained} new)`,
|
||||
);
|
||||
} else {
|
||||
console.log(`\n ✗ No healthy images found`);
|
||||
}
|
||||
|
||||
// Update seen-URLs cache
|
||||
seenUrlsCache[HEALTHY_CLASS] = Array.from(healthySeen);
|
||||
saveSeenUrlsCache(seenUrlsCache);
|
||||
}
|
||||
|
||||
// ── Summary ──────────────────────────────────────────────────────────────
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
const mins = Math.floor(elapsed / 60);
|
||||
const hrs = Math.floor(mins / 60);
|
||||
|
||||
// Final scan
|
||||
const finalScan = scanDataset();
|
||||
const totalHave = [...finalScan.diseaseCounts.values()].reduce((s, c) => s + c, 0);
|
||||
const atTarget = [...finalScan.diseaseCounts.values()].filter(
|
||||
(c) => c >= TARGET_PER_DISEASE,
|
||||
).length;
|
||||
|
||||
console.log("\n" + "=".repeat(60));
|
||||
console.log(" ✅ FILL COMPLETE");
|
||||
console.log("=".repeat(60));
|
||||
console.log(` Time: ${hrs}h ${mins % 60}m`);
|
||||
console.log(` Diseases at target: ${atTarget}/${diseaseInfo.size}`);
|
||||
console.log(` Total images: ${totalHave}`);
|
||||
console.log(` Healthy images: ${finalScan.healthyCount}/${TARGET_HEALTHY}`);
|
||||
console.log(` New downloads: ${totalDownloaded}`);
|
||||
console.log(` Dataset dir: ${DATASET_DIR}/`);
|
||||
|
||||
await closeDb();
|
||||
console.log("=".repeat(60));
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\nFatal error:", `\n${err}`);
|
||||
process.exit(1);
|
||||
});
|
||||
537
scripts/fine-tune-model.py
Normal file
537
scripts/fine-tune-model.py
Normal file
@@ -0,0 +1,537 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
fine-tune-model.py
|
||||
|
||||
Fine-tunes the PlantVillage MobileNetV2 model on a custom 95-class dataset
|
||||
(93 diseases + healthy + unknown).
|
||||
|
||||
Pipeline:
|
||||
1. Load `best_mnv2_pv_original.keras` (MobileNetV2 backbone + 38-class head)
|
||||
2. Replace the 38-class head with 95 classes (order matches diseases.json + healthy + unknown)
|
||||
3. Freeze backbone, train only the new classification head
|
||||
4. Unfreeze the last ~20 layers, fine-tune at lower learning rate
|
||||
5. Export to TF.js GraphModel format
|
||||
6. Export to .keras for future retraining
|
||||
|
||||
Usage: .tfjs-venv/bin/python scripts/fine-tune-model.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # Suppress TF info/warnings
|
||||
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
import keras
|
||||
from keras import layers, optimizers, regularizers
|
||||
|
||||
# ─── Constants ───────────────────────────────────────────────────────────────
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
MODEL_PATH = (
|
||||
PROJECT_ROOT
|
||||
/ "public"
|
||||
/ "models"
|
||||
/ "plant-disease-classifier"
|
||||
/ "best_mnv2_pv_original.keras"
|
||||
)
|
||||
DISEASES_JSON = PROJECT_ROOT / "src" / "data" / "diseases.json"
|
||||
DATASET_DIR = PROJECT_ROOT / "data" / "dataset"
|
||||
OUTPUT_DIR = PROJECT_ROOT / "public" / "models" / "plant-disease-classifier"
|
||||
TFJS_OUTPUT = OUTPUT_DIR / "tfjs_finetuned"
|
||||
|
||||
IMG_SIZE = 160 # Model input size
|
||||
BATCH_SIZE = 32
|
||||
EPOCHS_HEAD = 15 # Train just the new head
|
||||
EPOCHS_FINETUNE = 10 # Unfreeze and fine-tune
|
||||
LEARNING_RATE_HEAD = 1e-3
|
||||
LEARNING_RATE_FINETUNE = 1e-5
|
||||
VALIDATION_SPLIT = 0.15
|
||||
|
||||
NUM_CLASSES = 95 # healthy(0) + 93 diseases + unknown(94)
|
||||
|
||||
# ─── Class Mapping ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_class_mapping():
|
||||
"""
|
||||
Build a dict mapping dataset directory names → model class indices.
|
||||
Matches the ordering in labels.ts / diseases.json.
|
||||
|
||||
Index 0 = "healthy"
|
||||
Index 1-93 = disease IDs (in diseases.json order)
|
||||
Index 94 = "unknown" (no images — skip during training)
|
||||
"""
|
||||
with open(DISEASES_JSON) as f:
|
||||
diseases = json.load(f)
|
||||
|
||||
mapping = {"healthy": 0}
|
||||
for i, disease in enumerate(diseases):
|
||||
mapping[disease["id"]] = i + 1 # Index 1-93
|
||||
mapping["unknown"] = 94 # Not trained, but reserved
|
||||
|
||||
# Reverse mapping for predictions
|
||||
index_to_class = {v: k for k, v in mapping.items()}
|
||||
|
||||
return mapping, index_to_class
|
||||
|
||||
|
||||
def verify_dataset(mapping):
|
||||
"""Find which classes have images and how many."""
|
||||
available = {}
|
||||
total = 0
|
||||
|
||||
for class_id, class_idx in mapping.items():
|
||||
class_dir = DATASET_DIR / class_id
|
||||
if not class_dir.exists():
|
||||
continue
|
||||
|
||||
image_paths = sorted(class_dir.glob("*"))
|
||||
image_paths = [
|
||||
p
|
||||
for p in image_paths
|
||||
if p.suffix.lower() in (".jpg", ".jpeg", ".png", ".webp")
|
||||
]
|
||||
|
||||
if image_paths:
|
||||
available[class_id] = {"index": class_idx, "count": len(image_paths)}
|
||||
total += len(image_paths)
|
||||
|
||||
return available, total
|
||||
|
||||
|
||||
def print_dataset_summary(available, total):
|
||||
"""Print a summary of what's available."""
|
||||
print(f"\n{'─' * 60}")
|
||||
print("DATASET SUMMARY")
|
||||
print(f"{'─' * 60}")
|
||||
print(f" Total images: {total}")
|
||||
print(f" Classes found: {len(available)} / {len(build_class_mapping()[0])}")
|
||||
print(
|
||||
f" Missing classes with no images: {len(build_class_mapping()[0]) - len(available)}"
|
||||
)
|
||||
|
||||
# Count images per class
|
||||
counts = [(v["index"], k, v["count"]) for k, v in available.items()]
|
||||
counts.sort(key=lambda x: x[1])
|
||||
|
||||
print("\n Images per class:")
|
||||
for idx, class_id, count in counts:
|
||||
label = f" {idx:3d}. {class_id:<35s} {count:>4d} images"
|
||||
if class_id == "healthy":
|
||||
label += " ← 2× target"
|
||||
print(label)
|
||||
|
||||
# Stats
|
||||
class_counts = [v["count"] for v in available.values()]
|
||||
if class_counts:
|
||||
print(
|
||||
f"\n Min: {min(class_counts)} Max: {max(class_counts)} Avg: {sum(class_counts) / len(class_counts):.0f}"
|
||||
)
|
||||
print(f"{'─' * 60}\n")
|
||||
|
||||
|
||||
# ─── Data Loading ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def load_dataset(mapping, available):
|
||||
"""
|
||||
Load images from the dataset directory.
|
||||
Returns train/validation datasets with augmentation.
|
||||
"""
|
||||
# Build file paths and labels
|
||||
file_paths = []
|
||||
labels = []
|
||||
|
||||
for class_id, info in available.items():
|
||||
class_dir = DATASET_DIR / class_id
|
||||
images = sorted(class_dir.glob("*"))
|
||||
images = [
|
||||
p for p in images if p.suffix.lower() in (".jpg", ".jpeg", ".png", ".webp")
|
||||
]
|
||||
|
||||
for img_path in images:
|
||||
file_paths.append(str(img_path))
|
||||
labels.append(info["index"])
|
||||
|
||||
file_paths = np.array(file_paths)
|
||||
labels = np.array(labels)
|
||||
|
||||
# Shuffle
|
||||
indices = np.random.RandomState(42).permutation(len(file_paths))
|
||||
file_paths = file_paths[indices]
|
||||
labels = labels[indices]
|
||||
|
||||
# Split train/validation
|
||||
split = int(len(file_paths) * (1 - VALIDATION_SPLIT))
|
||||
train_paths, val_paths = file_paths[:split], file_paths[split:]
|
||||
train_labels, val_labels = labels[:split], labels[split:]
|
||||
|
||||
print(f" Train: {len(train_paths)} images")
|
||||
print(f" Val: {len(val_paths)} images")
|
||||
|
||||
# Parse function
|
||||
def parse_image(image_path, label):
|
||||
img = tf.io.read_file(image_path)
|
||||
img = tf.image.decode_image(img, channels=3, expand_animations=False)
|
||||
img = tf.image.resize(img, [IMG_SIZE, IMG_SIZE])
|
||||
img = tf.cast(img, tf.float32) / 255.0
|
||||
# ImageNet normalization (matching training-time preprocessing)
|
||||
mean = tf.constant([0.485, 0.456, 0.406])
|
||||
std = tf.constant([0.229, 0.224, 0.225])
|
||||
img = (img - mean) / std
|
||||
return img, label
|
||||
|
||||
def augment(image, label):
|
||||
"""Data augmentation for training set."""
|
||||
# Random horizontal flip
|
||||
image = tf.image.random_flip_left_right(image)
|
||||
# Random rotation (±20°)
|
||||
image = tf.image.random_flip_up_down(image)
|
||||
# Random brightness
|
||||
image = tf.image.random_brightness(image, 0.15)
|
||||
# Random contrast
|
||||
image = tf.image.random_contrast(image, 0.8, 1.2)
|
||||
# Random saturation
|
||||
image = tf.image.random_saturation(image, 0.8, 1.2)
|
||||
# Random hue
|
||||
image = tf.image.random_hue(image, 0.05)
|
||||
# Random crop (after slightly scaling up)
|
||||
image = tf.image.resize_with_crop_or_pad(image, IMG_SIZE + 12, IMG_SIZE + 12)
|
||||
image = tf.image.resize(image, [IMG_SIZE, IMG_SIZE])
|
||||
# Clip to valid range after augmentations
|
||||
image = tf.clip_by_value(image, -2.5, 2.5)
|
||||
return image, label
|
||||
|
||||
# Create tf.data datasets
|
||||
train_ds = tf.data.Dataset.from_tensor_slices((train_paths, train_labels))
|
||||
train_ds = train_ds.map(parse_image, num_parallel_calls=tf.data.AUTOTUNE)
|
||||
train_ds = train_ds.map(augment, num_parallel_calls=tf.data.AUTOTUNE)
|
||||
train_ds = train_ds.shuffle(1000).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)
|
||||
|
||||
val_ds = tf.data.Dataset.from_tensor_slices((val_paths, val_labels))
|
||||
val_ds = val_ds.map(parse_image, num_parallel_calls=tf.data.AUTOTUNE)
|
||||
val_ds = val_ds.batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)
|
||||
|
||||
return train_ds, val_ds
|
||||
|
||||
|
||||
# ─── Model Building ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_model():
|
||||
"""
|
||||
Load the PlantVillage model and replace the classification head
|
||||
with a 95-class output.
|
||||
"""
|
||||
print(f"\nLoading base model from: {MODEL_PATH}")
|
||||
if not MODEL_PATH.exists():
|
||||
print(f"ERROR: Model not found at {MODEL_PATH}")
|
||||
sys.exit(1)
|
||||
|
||||
base_model = keras.models.load_model(str(MODEL_PATH))
|
||||
print(f" Base model loaded: {type(base_model).__name__}")
|
||||
print(f" Input shape: {base_model.input_shape}")
|
||||
print(f" Output shape: {base_model.output_shape}")
|
||||
|
||||
# Extract backbone — everything up to the GlobalAveragePooling2D
|
||||
# The model structure is:
|
||||
# input_layer_2 → mobilenetv2_1.00_160 → global_average_pooling2d → dropout → dense(38)
|
||||
backbone_output = base_model.get_layer("global_average_pooling2d").output
|
||||
print(" Using backbone output: global_average_pooling2d")
|
||||
|
||||
# Freeze all backbone layers initially
|
||||
# (we'll unfreeze later for fine-tuning)
|
||||
for layer in base_model.layers:
|
||||
if layer.name != "dense": # We'll replace this anyway
|
||||
layer.trainable = False
|
||||
|
||||
# Build new classification head
|
||||
x = backbone_output
|
||||
x = layers.Dropout(0.3, name="dropout_new")(x)
|
||||
x = layers.Dense(
|
||||
NUM_CLASSES,
|
||||
activation="softmax",
|
||||
name="dense_new",
|
||||
kernel_regularizer=regularizers.l2(1e-4),
|
||||
)(x)
|
||||
|
||||
# Create new model
|
||||
model = keras.Model(
|
||||
inputs=base_model.input, outputs=x, name="plant-disease-classifier-v2"
|
||||
)
|
||||
|
||||
print(f" New model input: {model.input_shape}")
|
||||
print(f" New model output: {model.output_shape} ({NUM_CLASSES} classes)")
|
||||
|
||||
# Count trainable params
|
||||
backbone_trainable = sum(
|
||||
w.shape.num_elements()
|
||||
for layer in base_model.layers
|
||||
if layer.name != "dense"
|
||||
for w in layer.trainable_weights
|
||||
)
|
||||
head_trainable = sum(
|
||||
w.shape.num_elements() for w in model.get_layer("dense_new").trainable_weights
|
||||
)
|
||||
|
||||
print(f" Backbone frozen: {backbone_trainable:,} params (not training)")
|
||||
print(f" New head: {head_trainable:,} params (training)")
|
||||
|
||||
return model
|
||||
|
||||
|
||||
# ─── Training ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def train_head(model, train_ds, val_ds):
|
||||
"""Stage 1: Train only the new classification head."""
|
||||
print(f"\n{'=' * 60}")
|
||||
print("STAGE 1: Training classification head")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Epochs: {EPOCHS_HEAD}")
|
||||
print(f" Learning rate: {LEARNING_RATE_HEAD}")
|
||||
print(f" Batch size: {BATCH_SIZE}")
|
||||
|
||||
# Freeze all backbone layers
|
||||
for layer in model.layers:
|
||||
if layer.name != "dense_new":
|
||||
layer.trainable = False
|
||||
else:
|
||||
layer.trainable = True
|
||||
|
||||
# Verify
|
||||
trainable = sum(w.shape.num_elements() for w in model.trainable_weights)
|
||||
total = sum(w.shape.num_elements() for w in model.weights)
|
||||
print(f" Trainable params: {trainable:,} / {total:,} total")
|
||||
|
||||
model.compile(
|
||||
optimizer=optimizers.Adam(learning_rate=LEARNING_RATE_HEAD),
|
||||
loss="sparse_categorical_crossentropy",
|
||||
metrics=["accuracy", "sparse_top_k_categorical_accuracy"],
|
||||
)
|
||||
|
||||
history = model.fit(
|
||||
train_ds,
|
||||
validation_data=val_ds,
|
||||
epochs=EPOCHS_HEAD,
|
||||
verbose=1,
|
||||
callbacks=[
|
||||
keras.callbacks.EarlyStopping(
|
||||
monitor="val_accuracy",
|
||||
patience=3,
|
||||
restore_best_weights=True,
|
||||
),
|
||||
keras.callbacks.ReduceLROnPlateau(
|
||||
monitor="val_loss",
|
||||
factor=0.5,
|
||||
patience=2,
|
||||
min_lr=1e-6,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
final_val_acc = history.history["val_accuracy"][-1]
|
||||
print(f"\n Stage 1 complete! Val accuracy: {final_val_acc:.4f}")
|
||||
return history
|
||||
|
||||
|
||||
def train_finetune(model, train_ds, val_ds):
|
||||
"""Stage 2: Unfreeze last ~25 layers and fine-tune."""
|
||||
print(f"\n{'=' * 60}")
|
||||
print("STAGE 2: Fine-tuning backbone (last ~25 layers)")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Epochs: {EPOCHS_FINETUNE}")
|
||||
print(f" Learning rate: {LEARNING_RATE_FINETUNE}")
|
||||
|
||||
# Find the MobileNetV2 functional module
|
||||
# The backbone is a Functional model inside the base model
|
||||
mobilenet_layer = model.get_layer("mobilenetv2_1.00_160")
|
||||
|
||||
# Unfreeze the last ~25 layers of the backbone
|
||||
total_backbone_layers = len(mobilenet_layer.layers)
|
||||
unfreeze_from = max(0, total_backbone_layers - 25)
|
||||
print(
|
||||
f" Backbone has {total_backbone_layers} layers, unfreezing from layer {unfreeze_from}"
|
||||
)
|
||||
|
||||
for i, layer in enumerate(mobilenet_layer.layers):
|
||||
layer.trainable = i >= unfreeze_from
|
||||
|
||||
# Also unfreeze the new head
|
||||
model.get_layer("dense_new").trainable = True
|
||||
model.get_layer("dropout_new").trainable = True
|
||||
|
||||
trainable = sum(w.shape.num_elements() for w in model.trainable_weights)
|
||||
total = sum(w.shape.num_elements() for w in model.weights)
|
||||
print(f" Trainable params: {trainable:,} / {total:,} total")
|
||||
|
||||
model.compile(
|
||||
optimizer=optimizers.Adam(learning_rate=LEARNING_RATE_FINETUNE),
|
||||
loss="sparse_categorical_crossentropy",
|
||||
metrics=["accuracy", "sparse_top_k_categorical_accuracy"],
|
||||
)
|
||||
|
||||
history = model.fit(
|
||||
train_ds,
|
||||
validation_data=val_ds,
|
||||
epochs=EPOCHS_FINETUNE,
|
||||
verbose=1,
|
||||
callbacks=[
|
||||
keras.callbacks.EarlyStopping(
|
||||
monitor="val_accuracy",
|
||||
patience=3,
|
||||
restore_best_weights=True,
|
||||
),
|
||||
keras.callbacks.ReduceLROnPlateau(
|
||||
monitor="val_loss",
|
||||
factor=0.5,
|
||||
patience=2,
|
||||
min_lr=1e-7,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
final_val_acc = history.history["val_accuracy"][-1]
|
||||
print(f"\n Stage 2 complete! Val accuracy: {final_val_acc:.4f}")
|
||||
return history
|
||||
|
||||
|
||||
# ─── Export ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def export_models(model, class_mapping, index_to_class):
|
||||
"""Export the trained model to .keras and TF.js formats."""
|
||||
print(f"\n{'=' * 60}")
|
||||
print("EXPORTING")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
# 1. Save as .keras (for future retraining)
|
||||
keras_path = OUTPUT_DIR / "model-finetuned.keras"
|
||||
model.save(str(keras_path))
|
||||
print(f" ✓ Saved .keras: {keras_path}")
|
||||
|
||||
# 2. Save class mapping alongside the model
|
||||
mapping_path = OUTPUT_DIR / "class_mapping.json"
|
||||
with open(mapping_path, "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
"index_to_class": index_to_class,
|
||||
"class_to_index": class_mapping,
|
||||
"num_classes": NUM_CLASSES,
|
||||
"input_size": IMG_SIZE,
|
||||
},
|
||||
f,
|
||||
indent=2,
|
||||
)
|
||||
print(f" ✓ Saved class mapping: {mapping_path}")
|
||||
|
||||
# 3. Export to TF.js format
|
||||
tfjs_path = str(TFJS_OUTPUT)
|
||||
if TFJS_OUTPUT.exists():
|
||||
shutil.rmtree(tfjs_path)
|
||||
|
||||
try:
|
||||
import tensorflowjs as tfjs
|
||||
|
||||
tfjs.converters.save_keras_model(model, tfjs_path)
|
||||
print(f" ✓ Saved TF.js: {tfjs_path}/")
|
||||
for f in sorted(TFJS_OUTPUT.iterdir()):
|
||||
size = f.stat().st_size
|
||||
print(f" {f.name:<30s} {size:>10,} bytes")
|
||||
except Exception as e:
|
||||
print(f" ⚠ TF.js export failed: {e}")
|
||||
print(
|
||||
f" Run later: tensorflowjs_converter --input_format=keras {keras_path} {tfjs_path}"
|
||||
)
|
||||
|
||||
|
||||
# ─── Cleanup Old Model Files ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def cleanup_old_model():
|
||||
"""Remove old model.json and shards from the directory."""
|
||||
for f in OUTPUT_DIR.glob("model.json"):
|
||||
print(f" Removing old: {f.name}")
|
||||
f.unlink()
|
||||
for f in OUTPUT_DIR.glob("group1-shard*"):
|
||||
print(f" Removing old: {f.name}")
|
||||
f.unlink()
|
||||
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("PLANT DISEASE MODEL FINE-TUNER")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. Build class mapping
|
||||
print("\n[1/5] Building class mapping...")
|
||||
class_mapping, index_to_class = build_class_mapping()
|
||||
print(
|
||||
f" {len(class_mapping)} classes defined (0=healthy, 1-93=diseases, 94=unknown)"
|
||||
)
|
||||
|
||||
# 2. Verify dataset
|
||||
print("\n[2/5] Verifying dataset...")
|
||||
if not DATASET_DIR.exists():
|
||||
print(f" ERROR: Dataset not found at {DATASET_DIR}")
|
||||
print(" Run the scraper first: npx tsx scripts/scrape-training-dataset.ts")
|
||||
sys.exit(1)
|
||||
|
||||
available, total = verify_dataset(class_mapping)
|
||||
print_dataset_summary(available, total)
|
||||
|
||||
if total < 100:
|
||||
print(f" WARNING: Only {total} images. Consider scraping more data.")
|
||||
print(" Continue anyway? (y/n)")
|
||||
# Continue regardless — user can decide
|
||||
|
||||
# 3. Load dataset
|
||||
print("\n[3/5] Loading and augmenting dataset...")
|
||||
train_ds, val_ds = load_dataset(class_mapping, available)
|
||||
|
||||
# 4. Build and train model
|
||||
print("\n[4/5] Building model...")
|
||||
model = build_model()
|
||||
model.summary()
|
||||
|
||||
# Check if training should run
|
||||
if total > 0:
|
||||
train_head(model, train_ds, val_ds)
|
||||
train_finetune(model, train_ds, val_ds)
|
||||
|
||||
# 5. Export
|
||||
print("\n[5/5] Exporting models...")
|
||||
cleanup_old_model()
|
||||
export_models(model, class_mapping, index_to_class)
|
||||
else:
|
||||
print("\n Skipping training — no dataset available.")
|
||||
sys.exit(1)
|
||||
|
||||
# ── Final Summary ────────────────────────────────────────────────────────
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("DONE! Model fine-tuned and exported.")
|
||||
print(f"{'=' * 60}")
|
||||
print("\nFiles created:")
|
||||
print(f" {OUTPUT_DIR / 'model-finetuned.keras'}")
|
||||
print(f" {OUTPUT_DIR / 'class_mapping.json'}")
|
||||
print(f" {TFJS_OUTPUT / 'model.json'}")
|
||||
print("\nTo update your app:")
|
||||
print(" 1. Replace model files:")
|
||||
print(f" cp {TFJS_OUTPUT / 'model.json'} {OUTPUT_DIR / 'model.json'}")
|
||||
print(f" cp {TFJS_OUTPUT / 'group1-shard*'} {OUTPUT_DIR / '/'}")
|
||||
print(" 2. Restart the dev server")
|
||||
print(" 3. Test with: POST /api/identify")
|
||||
print("\nNote: Update labels.ts if the class order changed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
212
scripts/fix-classifications.ts
Normal file
212
scripts/fix-classifications.ts
Normal file
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* fix-classifications.ts — Fix misclassified diseases in the DB.
|
||||
*
|
||||
* Fixes:
|
||||
* 1. Diseases named with viral indicators (mosaic, mottle, ringspot, virus, etc.)
|
||||
* that are incorrectly tagged as "fungal"
|
||||
* 2. Other suspicious patterns
|
||||
*
|
||||
* Usage: cd apps/web && npx tsx scripts/fix-classifications.ts
|
||||
*/
|
||||
|
||||
import { readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
|
||||
// Manually load .env.development
|
||||
const envPath = resolve(__dirname, "../.env.development");
|
||||
try {
|
||||
const env = readFileSync(envPath, "utf-8");
|
||||
for (const line of env.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed && !trimmed.startsWith("#")) {
|
||||
const eqIdx = trimmed.indexOf("=");
|
||||
if (eqIdx > 0) {
|
||||
const key = trimmed.slice(0, eqIdx).trim();
|
||||
const val = trimmed.slice(eqIdx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
import { getDb, closeDb } from "../src/lib/db/index";
|
||||
import { diseases } from "../src/lib/db/schema";
|
||||
import { createClient } from "@libsql/client";
|
||||
|
||||
type AgentType = "fungal" | "bacterial" | "viral" | "environmental";
|
||||
|
||||
interface FixRule {
|
||||
test: (name: string) => boolean;
|
||||
correctAgent: AgentType;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
const FIX_RULES: FixRule[] = [
|
||||
// Diseases explicitly named as "virus" or "viral"
|
||||
{
|
||||
test: (name) => /\b(virus|viral|viroid)\b/i.test(name),
|
||||
correctAgent: "viral",
|
||||
reason: "Name explicitly indicates viral disease",
|
||||
},
|
||||
// Potexvirus, carlavirus, etc.
|
||||
{
|
||||
test: (name) =>
|
||||
/\b(virus\b|potex|carla|tobamo|poty|cucumo|ilar|nepo|tymovirus|geminivir|tom bushy stunt)\b/i.test(
|
||||
name,
|
||||
),
|
||||
correctAgent: "viral",
|
||||
reason: "Recognized virus genus in name",
|
||||
},
|
||||
// "Mosaic" diseases (typically viral)
|
||||
{
|
||||
test: (name) => /\bmosaic\b/i.test(name),
|
||||
correctAgent: "viral",
|
||||
reason: "Mosaic symptoms are typically caused by viruses",
|
||||
},
|
||||
// "Mottle" diseases (typically viral)
|
||||
{
|
||||
test: (name) => /\bmottle\b/i.test(name),
|
||||
correctAgent: "viral",
|
||||
reason: "Mottle symptoms are typically caused by viruses",
|
||||
},
|
||||
// "Ringspot" diseases (typically viral)
|
||||
{
|
||||
test: (name) => /\bringspot\b/i.test(name),
|
||||
correctAgent: "viral",
|
||||
reason: "Ringspot symptoms are typically caused by viruses",
|
||||
},
|
||||
// "Leaf curl" (many are viral)
|
||||
{
|
||||
test: (name) => /\bleaf curl\b|\bleafroll\b|\bleaf-roll\b/i.test(name),
|
||||
correctAgent: "viral",
|
||||
reason: "Leaf curl/roll diseases are often viral",
|
||||
},
|
||||
// "Rosette" (often viral or phytoplasma)
|
||||
{
|
||||
test: (name) => /\brosette\b/i.test(name),
|
||||
correctAgent: "viral",
|
||||
reason: "Rosette diseases are typically viral or phytoplasma",
|
||||
},
|
||||
// "Yellows" (often phytoplasma/viral)
|
||||
{
|
||||
test: (name) => /\byellows\b/i.test(name) && !/\bpeach\b/i.test(name),
|
||||
correctAgent: "viral",
|
||||
reason: "Yellows diseases are typically phytoplasma or viral",
|
||||
},
|
||||
// "Stunt" / "Dwarf" (often viral)
|
||||
{
|
||||
test: (name) => /\b(stunt|dwarf(ism)?)\b/i.test(name),
|
||||
correctAgent: "viral",
|
||||
reason: "Stunting/dwarfing diseases are often viral",
|
||||
},
|
||||
// Explicit bacterial in name
|
||||
{
|
||||
test: (name) =>
|
||||
/\bbacterial\b|\bbacterium\b|\berwinia\b|\bpseudomonas\b|\bxanthomonas\b|\bralstonia\b|\bclavibacter\b|\bstreptomyces\b|\bagrobacterium\b/i.test(
|
||||
name,
|
||||
),
|
||||
correctAgent: "bacterial",
|
||||
reason: "Name indicates bacterial disease",
|
||||
},
|
||||
// Environmental/abiotic indicators
|
||||
{
|
||||
test: (name) =>
|
||||
/\b(deficiency|abiotic|environmental|injury|damage|stress|sunscald|sunburn|chilling|freeze|frost|wind|hail|nutrient|toxicity|snow\s+(mold|scald)|winter\s+(injury|rot|kill))\b/i.test(
|
||||
name,
|
||||
),
|
||||
correctAgent: "environmental",
|
||||
reason: "Name indicates abiotic/environmental cause",
|
||||
},
|
||||
];
|
||||
|
||||
async function main() {
|
||||
console.log("🔍 Fixing disease classifications\n");
|
||||
const db = getDb();
|
||||
const allDiseases = await db
|
||||
.select({ id: diseases.id, name: diseases.name, causalAgentType: diseases.causalAgentType })
|
||||
.from(diseases)
|
||||
.all();
|
||||
console.log(`📋 ${allDiseases.length} total diseases\n`);
|
||||
|
||||
const rawClient = createClient({
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
});
|
||||
|
||||
const updates: { id: string; newAgent: AgentType; rule: FixRule; oldAgent: string }[] = [];
|
||||
|
||||
for (const d of allDiseases) {
|
||||
for (const rule of FIX_RULES) {
|
||||
if (rule.test(d.name)) {
|
||||
if (d.causalAgentType !== rule.correctAgent) {
|
||||
updates.push({
|
||||
id: d.id,
|
||||
newAgent: rule.correctAgent,
|
||||
rule,
|
||||
oldAgent: d.causalAgentType,
|
||||
});
|
||||
}
|
||||
break; // First matching rule wins
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Found ${updates.length} diseases needing reclassification:\n`);
|
||||
|
||||
// Group by correction type
|
||||
const grouped: Record<string, { from: string; to: string; items: string[] }> = {};
|
||||
for (const u of updates) {
|
||||
const key = `${u.oldAgent}→${u.newAgent}`;
|
||||
if (!grouped[key]) grouped[key] = { from: u.oldAgent, to: u.newAgent, items: [] };
|
||||
grouped[key].items.push(` ${u.id}`);
|
||||
}
|
||||
|
||||
for (const [, g] of Object.entries(grouped)) {
|
||||
console.log(`${g.from} → ${g.to} (${g.items.length} diseases):`);
|
||||
g.items.slice(0, 10).forEach((l) => console.log(l));
|
||||
if (g.items.length > 10) console.log(` ... and ${g.items.length - 10} more`);
|
||||
console.log();
|
||||
}
|
||||
|
||||
// Apply updates
|
||||
if (updates.length === 0) {
|
||||
console.log("✅ No corrections needed");
|
||||
} else {
|
||||
console.log(`Applying ${updates.length} corrections...\n`);
|
||||
|
||||
// Batch update in groups of 50
|
||||
for (let i = 0; i < updates.length; i += 50) {
|
||||
const batch = updates.slice(i, i + 50);
|
||||
await rawClient.batch(
|
||||
batch.map((u) => ({
|
||||
sql: "UPDATE diseases SET causal_agent_type = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
args: [u.newAgent, u.id],
|
||||
})),
|
||||
"write",
|
||||
);
|
||||
process.stdout.write(` ${Math.min(i + 50, updates.length)}/${updates.length}\n`);
|
||||
}
|
||||
|
||||
console.log(`\n✅ ${updates.length} diseases reclassified`);
|
||||
}
|
||||
|
||||
// Print summary stats
|
||||
const after = await db.select({ causalAgentType: diseases.causalAgentType }).from(diseases).all();
|
||||
const counts: Record<string, number> = {};
|
||||
after.forEach((d) => {
|
||||
counts[d.causalAgentType] = (counts[d.causalAgentType] || 0) + 1;
|
||||
});
|
||||
console.log("\n📊 Updated distribution:");
|
||||
for (const [type, count] of Object.entries(counts).sort()) {
|
||||
console.log(` ${type}: ${count}`);
|
||||
}
|
||||
|
||||
rawClient.close();
|
||||
closeDb();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("\n❌", err);
|
||||
process.exit(1);
|
||||
});
|
||||
385
scripts/generate-flagged-report.ts
Normal file
385
scripts/generate-flagged-report.ts
Normal file
@@ -0,0 +1,385 @@
|
||||
/**
|
||||
* generate-flagged-report.ts
|
||||
*
|
||||
* Reads all flagged content from the database and generates a pretty
|
||||
* markdown report organized by content type. The report includes:
|
||||
* - Summary table with counts per content type
|
||||
* - Plant images flagged for review
|
||||
* - Disease images flagged for review
|
||||
* - Disease symptoms flagged for review
|
||||
* - Disease causes flagged for review
|
||||
* - Disease treatment steps flagged for review
|
||||
* - Disease prevention tips flagged for review
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/generate-flagged-report.ts [--min-flags N] [--output path/to/report.md]
|
||||
*
|
||||
* Options:
|
||||
* --min-flags Minimum flag count to include (default: 1)
|
||||
* --output Output path (default: scripts/.flagged-content-review-needed.md)
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import path from "node:path";
|
||||
|
||||
// Load DB config from .env.development (or .env.production if NODE_ENV=production)
|
||||
const envFile =
|
||||
process.env.NODE_ENV === "production" ? "../.env.production" : "../.env.development";
|
||||
dotenv.config({ path: path.resolve(__dirname, envFile) });
|
||||
import { createClient } from "@libsql/client";
|
||||
import fs from "node:fs";
|
||||
|
||||
// ─── Config ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const MIN_FLAGS = parseInt(
|
||||
process.argv.find((a) => a.startsWith("--min-flags="))?.split("=")[1] ?? "1",
|
||||
10,
|
||||
);
|
||||
const OUTPUT_PATH =
|
||||
process.argv.find((a) => a.startsWith("--output="))?.split("=")[1] ??
|
||||
path.join(__dirname, ".flagged-content-review-needed.md");
|
||||
|
||||
// ─── DB Connection ──────────────────────────────────────────────────────────
|
||||
|
||||
const db = createClient({
|
||||
url: process.env.DATABASE_URL!,
|
||||
authToken: process.env.DATABASE_TOKEN!,
|
||||
});
|
||||
|
||||
// ─── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
interface FlaggedRow {
|
||||
id: string;
|
||||
content_type: string;
|
||||
content_id: string;
|
||||
field_name: string;
|
||||
notes: string;
|
||||
flag_count: number;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
interface PlantRow {
|
||||
id: string;
|
||||
common_name: string;
|
||||
scientific_name: string;
|
||||
family: string;
|
||||
image_url: string;
|
||||
}
|
||||
|
||||
interface DiseaseRow {
|
||||
id: string;
|
||||
name: string;
|
||||
scientific_name: string;
|
||||
plant_id: string;
|
||||
image_url: string;
|
||||
}
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
const CONTENT_TYPE_LABELS: Record<string, { emoji: string; title: string; description: string }> = {
|
||||
plant_image: {
|
||||
emoji: "🪴",
|
||||
title: "Plant Images Flagged for Review",
|
||||
description: "Plant images that users have flagged as potentially incorrect or low quality.",
|
||||
},
|
||||
disease_image: {
|
||||
emoji: "📸",
|
||||
title: "Disease Images Flagged for Review",
|
||||
description:
|
||||
"Disease symptom images that users have flagged as potentially incorrect or misleading.",
|
||||
},
|
||||
disease_description: {
|
||||
emoji: "📝",
|
||||
title: "Disease Descriptions Flagged for Review",
|
||||
description: "Disease descriptions that users have flagged as potentially inaccurate.",
|
||||
},
|
||||
disease_symptoms: {
|
||||
emoji: "⚠️",
|
||||
title: "Disease Symptoms Flagged for Review",
|
||||
description: "Symptom descriptions that users have flagged as potentially inaccurate.",
|
||||
},
|
||||
disease_causes: {
|
||||
emoji: "🔍",
|
||||
title: "Disease Causes Flagged for Review",
|
||||
description:
|
||||
"Causes and contributing factors that users have flagged as potentially incorrect.",
|
||||
},
|
||||
disease_treatment: {
|
||||
emoji: "💊",
|
||||
title: "Disease Treatment Steps Flagged for Review",
|
||||
description:
|
||||
"Treatment instructions that users have flagged as potentially incorrect or harmful.",
|
||||
},
|
||||
disease_prevention: {
|
||||
emoji: "🛡️",
|
||||
title: "Disease Prevention Tips Flagged for Review",
|
||||
description: "Prevention tips that users have flagged as potentially incorrect or misleading.",
|
||||
},
|
||||
};
|
||||
|
||||
function formatDate(iso: string): string {
|
||||
const d = new Date(iso);
|
||||
return d.toLocaleDateString("en-US", {
|
||||
year: "numeric",
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
console.log(`📋 Generating flagged content report (min flags: ${MIN_FLAGS})...`);
|
||||
|
||||
// Fetch flagged content
|
||||
const flaggedRs = await db.execute({
|
||||
sql: "SELECT * FROM flagged_content WHERE flag_count >= ? ORDER BY content_type, flag_count DESC, updated_at DESC",
|
||||
args: [MIN_FLAGS],
|
||||
});
|
||||
const flaggedRows = flaggedRs.rows as unknown as FlaggedRow[];
|
||||
|
||||
if (flaggedRows.length === 0) {
|
||||
const report = [
|
||||
"# 🚩 Flagged Content Review — Nothing to Review",
|
||||
"",
|
||||
`Generated: ${new Date().toISOString()}`,
|
||||
"",
|
||||
"**No content has been flagged for review yet.**",
|
||||
"",
|
||||
"Flagged items will appear here once users flag content for manual review.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
`_Report generated with min-flags=${MIN_FLAGS}_`,
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
fs.writeFileSync(OUTPUT_PATH, report, "utf-8");
|
||||
console.log(`✅ Report written to ${OUTPUT_PATH} (no flagged items)`);
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
// Collect all unique plant and disease IDs
|
||||
const plantIds = new Set<string>();
|
||||
const diseaseIds = new Set<string>();
|
||||
|
||||
for (const row of flaggedRows) {
|
||||
if (row.content_type === "plant_image") {
|
||||
plantIds.add(row.content_id);
|
||||
} else {
|
||||
diseaseIds.add(row.content_id);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch plant names
|
||||
const plantMap = new Map<string, PlantRow>();
|
||||
if (plantIds.size > 0) {
|
||||
const plantRs = await db.execute({
|
||||
sql: `SELECT id, common_name, scientific_name, family, image_url FROM plants WHERE id IN (${[...plantIds].map(() => "?").join(",")})`,
|
||||
args: [...plantIds],
|
||||
});
|
||||
for (const row of plantRs.rows as unknown as PlantRow[]) {
|
||||
plantMap.set(row.id, row);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch disease names + their plant references
|
||||
const diseaseMap = new Map<string, DiseaseRow>();
|
||||
if (diseaseIds.size > 0) {
|
||||
const diseaseRs = await db.execute({
|
||||
sql: `SELECT id, name, scientific_name, plant_id, image_url FROM diseases WHERE id IN (${[...diseaseIds].map(() => "?").join(",")})`,
|
||||
args: [...diseaseIds],
|
||||
});
|
||||
for (const row of diseaseRs.rows as unknown as DiseaseRow[]) {
|
||||
diseaseMap.set(row.id, row);
|
||||
if (!plantMap.has(row.plant_id)) {
|
||||
plantIds.add(row.plant_id);
|
||||
}
|
||||
}
|
||||
// Fetch any missing plant references for diseases
|
||||
if (plantIds.size > 0) {
|
||||
const missingPlantIds = [...plantIds].filter((id) => !plantMap.has(id));
|
||||
if (missingPlantIds.length > 0) {
|
||||
const plantRs = await db.execute({
|
||||
sql: `SELECT id, common_name, scientific_name, family, image_url FROM plants WHERE id IN (${missingPlantIds.map(() => "?").join(",")})`,
|
||||
args: missingPlantIds,
|
||||
});
|
||||
for (const row of plantRs.rows as unknown as PlantRow[]) {
|
||||
plantMap.set(row.id, row);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Group by content type
|
||||
const groups: Record<string, FlaggedRow[]> = {};
|
||||
for (const row of flaggedRows) {
|
||||
if (!groups[row.content_type]) groups[row.content_type] = [];
|
||||
groups[row.content_type].push(row);
|
||||
}
|
||||
|
||||
// ─── Build Report ────────────────────────────────────────────────────────
|
||||
|
||||
const lines: string[] = [];
|
||||
const totalFlags = flaggedRows.reduce((sum, r) => sum + r.flag_count, 0);
|
||||
|
||||
lines.push("# 🚩 Flagged Content — Manual Review Needed");
|
||||
lines.push("");
|
||||
lines.push(`Generated: ${new Date().toISOString()}`);
|
||||
lines.push("");
|
||||
lines.push(
|
||||
flaggedRows.length === 1
|
||||
? `**${flaggedRows.length} item** flagged for review (${totalFlags} total flags).`
|
||||
: `**${flaggedRows.length} items** flagged for review (${totalFlags} total flags).`,
|
||||
);
|
||||
lines.push("");
|
||||
lines.push("Most data in this knowledge base is not reviewed by humans. ");
|
||||
lines.push("Items listed below have been flagged by users for manual review. ");
|
||||
lines.push("Please review each item and take appropriate action.");
|
||||
lines.push("");
|
||||
|
||||
// Summary table
|
||||
lines.push("## 📊 Summary");
|
||||
lines.push("");
|
||||
lines.push("| Content Type | Count | Total Flags |");
|
||||
lines.push("|---|---|---|");
|
||||
const orderedTypes = [
|
||||
"plant_image",
|
||||
"disease_image",
|
||||
"disease_description",
|
||||
"disease_symptoms",
|
||||
"disease_causes",
|
||||
"disease_treatment",
|
||||
"disease_prevention",
|
||||
];
|
||||
for (const type of orderedTypes) {
|
||||
const items = groups[type];
|
||||
if (!items) continue;
|
||||
const label = CONTENT_TYPE_LABELS[type]?.title ?? type;
|
||||
const count = items.length;
|
||||
const sumFlags = items.reduce((s, r) => s + r.flag_count, 0);
|
||||
lines.push(`| ${label} | ${count} | ${sumFlags} |`);
|
||||
}
|
||||
lines.push(`| **Total** | **${flaggedRows.length}** | **${totalFlags}** |`);
|
||||
lines.push("");
|
||||
lines.push("---");
|
||||
lines.push("");
|
||||
|
||||
// Detail sections per content type
|
||||
for (const type of orderedTypes) {
|
||||
const items = groups[type];
|
||||
if (!items) continue;
|
||||
|
||||
const config = CONTENT_TYPE_LABELS[type];
|
||||
lines.push(`## ${config?.emoji ?? "📋"} ${config?.title ?? type}`);
|
||||
lines.push("");
|
||||
| < | ||||