Mike/plant-disease-id
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prevelance data added

Browse Source
This commit is contained in:
Michael Freno
2026-06-07 12:06:41 -04:00
parent cc7b2a593a
commit 876c26968b
11 changed files with 2305 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

445
apps/web/scripts/scrape-training-dataset.ts
View File

@@ -4,10 +4,10 @@
*
* Collects a training dataset from DuckDuckGo, iNaturalist, and Wikimedia Commons.
*
* Targets (tiered by plant type):
* - Core plants (houseplants + common garden): 100 images per disease
* - Full set (all 11,498 DB diseases): 10 images per disease
* - Healthy: 400 images
* Target: Top 200 most common plant diseases (ranked by iNaturalist observation counts)
* - 200 images per disease
* - 200 healthy plant images
* - Processes 5 diseases in parallel with 30 concurrent downloads each
*
* Sources (all free, no API keys):
* 1. DB image_url — existing images already found
@@ -42,66 +42,30 @@ try {
import { getDb, closeDb } from "@/lib/db/index";
import { diseases } from "@/lib/db/schema";
import { sql } from "drizzle-orm";
// ─── Config ─────────────────────────────────────────────────────────────────
const DATASET_DIR = resolve(__dirname, "../data/dataset");
const PROGRESS_FILE = resolve(DATASET_DIR, ".progress.json");
/** Target images per disease for CORE plants */
const TARGET_CORE = 100;
/** Target images per disease */
const TARGET_PER_DISEASE = 200;
/** Target images per disease for the FULL set */
const TARGET_FULL = 10;
/** Number of diseases to target (most common first) */
const TARGET_DISEASE_COUNT = 200;
/** Target images for the "healthy" class */
const TARGET_HEALTHY = 400;
/** Core plants that get higher image targets */
const CORE_PLANTS = new Set([
// Houseplants
"monstera",
"pothos",
"snake-plant",
"peace-lily",
"orchid",
"succulent",
"fiddle-leaf-fig",
"aloe-vera",
"cactus",
"fern",
// Garden plants
"tomato",
"basil",
"rose",
"pepper",
"strawberry",
"cucumber",
"squash",
"lettuce",
"spinach",
"cabbage",
"lavender",
"mint",
"jasmine",
"sunflower",
"daisy",
"zucchini",
"bean",
"eggplant",
"chili",
// General disease patterns
"general",
]);
/** Delay between DuckDuckGo search API calls (ms) */
const SEARCH_DELAY = 1500;
/** Delay between image downloads (ms) */
const DOWNLOAD_DELAY = 100;
/** Max concurrent image downloads per disease */
const CONCURRENT_DOWNLOADS = 30;
/** Max concurrent downloads */
const CONCURRENT_DOWNLOADS = 10;
/** Number of diseases to process in parallel */
const DISEASE_CONCURRENCY = 5;
/** Minimum image size in bytes to accept */
const MIN_IMAGE_SIZE = 10_000; // 10KB
@@ -167,21 +131,246 @@ interface Progress {
// ─── DB Loading ──────────────────────────────────────────────────────────────
const INAT_CACHE_FILE = resolve(DATASET_DIR, ".inat-prevalence-cache.json");
/**
* Load all diseases from the database with their existing image URLs.
* Query iNaturalist for real-world prevalence of a disease.
* Returns observation count (higher = more common in the real world).
*/
async function getInatPrevalence(diseaseName: string, plantName?: string): Promise<number> {
try {
const headers = { "User-Agent": UA, Accept: "application/json" };
const signal = AbortSignal.timeout(10_000);
const baseUrl = "https://api.inaturalist.org/v1/observations";
// Tier 1: disease + plant name, research-grade, Plantae/Fungi/Chromista
// This is the most specific and reliable query — filters to relevant kingdoms
// and only counts community-verified observations.
if (plantName) {
const q = `${diseaseName} ${plantName}`;
const url =
`${baseUrl}?q=${encodeURIComponent(q)}` +
`&quality_grade=research` +
`&iconic_taxon_id=47126,47158,47686` +
`&photos_only=true&per_page=1`;
const res = await fetch(url, { headers, signal });
if (res.ok) {
const data = (await res.json()) as { total_results: number };
if ((data.total_results ?? 0) > 0) return data.total_results!;
}
}
// Fallback: disease name only, all quality grades (original behavior)
const url = `${baseUrl}?q=${encodeURIComponent(diseaseName.toLowerCase())}&photos_only=true&per_page=1`;
const res = await fetch(url, { headers, signal });
if (!res.ok) return 0;
const data = (await res.json()) as { total_results: number };
return data.total_results ?? 0;
} catch {
return 0;
}
}
/**
* Load prevalence data from cache or build it by querying iNaturalist.
* Caches results to avoid re-querying on every run.
*/
async function loadPrevalenceData(
uniqueNames: string[],
plantMap?: Map<string, string>,
): Promise<Map<string, number>> {
// Load cache if exists
let cache: Record<string, number> = {};
if (existsSync(INAT_CACHE_FILE)) {
try {
cache = JSON.parse(readFileSync(INAT_CACHE_FILE, "utf-8"));
} catch {}
}
const prevalenceMap = new Map<string, number>();
const toQuery: string[] = [];
// Check which names need querying
for (const name of uniqueNames) {
const key = name.toLowerCase();
if (key in cache) {
prevalenceMap.set(name, cache[key]);
} else {
toQuery.push(name);
}
}
if (toQuery.length > 0) {
console.log(`\n Querying iNaturalist for ${toQuery.length} disease prevalence scores...`);
let queried = 0;
for (const name of toQuery) {
const count = await getInatPrevalence(name, plantMap?.get(name));
const key = name.toLowerCase();
cache[key] = count;
prevalenceMap.set(name, count);
queried++;
// Save cache every 10 queries
if (queried % 10 === 0) {
writeFileSync(INAT_CACHE_FILE, JSON.stringify(cache, null, 2));
console.log(` Queried ${queried}/${toQuery.length}...`);
}
// Rate limit: ~100 req/min
await sleep(600);
}
// Final cache save
writeFileSync(INAT_CACHE_FILE, JSON.stringify(cache, null, 2));
console.log(` ✓ Queried ${queried} diseases, cached to ${INAT_CACHE_FILE}`);
}
return prevalenceMap;
}
/**
* Persist prevalence scores to the database and update prevalence enum.
* Maps observation counts to common/uncommon/rare based on thresholds.
*/
async function persistPrevalenceData(
db: ReturnType<typeof getDb>,
prevalenceMap: Map<string, number>,
): Promise<void> {
// Load all diseases to update
const allDiseases = await db
.select({
id: diseases.id,
name: diseases.name,
})
.from(diseases);
// Compute percentile-based thresholds from actual score distribution.
// Top 25% → common, bottom 25% → rare, middle 50% → uncommon.
// This guarantees meaningful classification regardless of absolute scale.
const scores = Array.from(prevalenceMap.values())
.filter((s) => s > 0)
.sort((a, b) => a - b);
const n = scores.length;
const commonThreshold = n > 0 ? scores[Math.floor(n * 0.75)] : 1000;
const rareThreshold = n > 0 ? scores[Math.floor(n * 0.25)] : 10;
console.log(
`\n Prevalence distribution: ${n} non-zero scores` +
`, p25=${rareThreshold.toLocaleString()}` +
`, p75=${commonThreshold.toLocaleString()}`,
);
console.log(` Persisting prevalence data for ${allDiseases.length} diseases...`);
let updated = 0;
for (const disease of allDiseases) {
const score = prevalenceMap.get(disease.name) ?? 0;
// Map score to prevalence enum using distribution-based thresholds.
// Score of 0 means no iNaturalist observations found — genuinely rare.
let prevalence: "common" | "uncommon" | "rare" | "very_rare";
if (score === 0) {
prevalence = "very_rare";
} else if (score >= commonThreshold) {
prevalence = "common";
} else if (score > rareThreshold) {
prevalence = "uncommon";
} else {
prevalence = "rare";
}
await db
.update(diseases)
.set({
prevalenceScore: score,
prevalence,
updatedAt: sql`(datetime('now'))`,
})
.where(sql`${diseases.id} = ${disease.id}`);
updated++;
if (updated % 100 === 0) {
console.log(` Updated ${updated}/${allDiseases.length}...`);
}
}
console.log(` ✓ Updated ${updated} diseases with prevalence data`);
}
/**
* Load the top 200 most common diseases from the database.
* Ranks by iNaturalist observation counts (real-world prevalence data).
*/
async function loadDiseasesFromDb(): Promise<DbDisease[]> {
const db = getDb();
const rows = await db
// Get unique disease names and their most common host plant for better iNaturalist queries.
const nameStats = await db
.select({
name: diseases.name,
plantId: diseases.plantId,
count: sql<number>`COUNT(*)`.mapWith(Number),
})
.from(diseases)
.groupBy(diseases.name, diseases.plantId);
// Aggregate: unique names, name frequency (across all plants), and most common plant per name
const seenNames = new Set<string>();
const nameFrequency = new Map<string, number>();
const plantFreq = new Map<string, Map<string, number>>();
let totalDiseases = 0;
for (const row of nameStats) {
seenNames.add(row.name);
nameFrequency.set(row.name, (nameFrequency.get(row.name) ?? 0) + row.count);
totalDiseases += row.count;
if (!plantFreq.has(row.name)) plantFreq.set(row.name, new Map());
plantFreq.get(row.name)!.set(row.plantId, row.count);
}
const uniqueNames = [...seenNames];
// For each disease name, pick the most frequent host plant for more specific iNaturalist queries
const plantMap = new Map<string, string>();
for (const [name, freq] of plantFreq) {
const top = [...freq.entries()].sort((a, b) => b[1] - a[1])[0];
plantMap.set(name, top[0]);
}
console.log(
` Found ${uniqueNames.length} unique disease names across ${totalDiseases} diseases`,
);
// Load or build prevalence data from iNaturalist (with plant context for better queries)
const prevalenceMap = await loadPrevalenceData(uniqueNames, plantMap);
// Persist prevalence scores to database
await persistPrevalenceData(db, prevalenceMap);
// Load all diseases
const allDiseases = await db
.select({
id: diseases.id,
plantId: diseases.plantId,
name: diseases.name,
imageUrl: diseases.imageUrl,
})
.from(diseases)
.orderBy(diseases.id);
return rows;
.from(diseases);
// Sort by iNaturalist prevalence (descending), then by name frequency as tiebreaker
allDiseases.sort((a, b) => {
const prevA = prevalenceMap.get(a.name) ?? 0;
const prevB = prevalenceMap.get(b.name) ?? 0;
if (prevA !== prevB) return prevB - prevA;
// Tiebreaker: name frequency
const freqA = nameFrequency.get(a.name) ?? 0;
const freqB = nameFrequency.get(b.name) ?? 0;
return freqB - freqA;
});
// Return top TARGET_DISEASE_COUNT
return allDiseases.slice(0, TARGET_DISEASE_COUNT);
}
// ─── DuckDuckGo API ─────────────────────────────────────────────────────────
@@ -208,7 +397,9 @@ async function searchImagesDuckDuckGo(
vqd: string,
page: number,
): Promise<DuckDuckGoImageResult[]> {
const url = `https://duckduckgo.com/i.js?q=${encodeURIComponent(query)}&vqd=${vqd}&o=json&p=${page}&f=,,,`;
const url = `https://duckduckgo.com/i.js?q=${encodeURIComponent(
query,
)}&vqd=${vqd}&o=json&p=${page}&f=,,,`;
const res = await fetch(url, {
headers: {
@@ -396,7 +587,9 @@ async function searchImagesCommons(
for (const hit of hits) {
if (results.length >= target) break;
const filename = hit.title.replace(/^File:/, "");
const imgUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(filename)}`;
const imgUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(
filename,
)}`;
if (seenUrls.has(imgUrl)) continue;
seenUrls.add(imgUrl);
results.push(imgUrl);
@@ -461,7 +654,6 @@ async function downloadBatch(
const paddedIndex = String(index).padStart(4, "0");
const destPath = resolve(classDir, `img_${paddedIndex}.jpg`);
const success = await downloadImage(url, destPath);
await sleep(DOWNLOAD_DELAY);
return { success, index: index++, url: url.substring(0, 50) };
}),
);
@@ -496,19 +688,36 @@ function loadProgress(): Progress {
}
try {
const raw = JSON.parse(readFileSync(PROGRESS_FILE, "utf-8")) as Partial<Progress>;
// Backward compat: ensure new fields exist
raw.phase ??= 0;
raw.phaseIndex ??= 0;
raw.classes ??= {};
// Migration: detect old tiered system (phaseIndex > 200 means it's from old core/full system)
const isOldFormat = (raw.phaseIndex ?? 0) > 200 || !raw.phase;
if (isOldFormat) {
console.warn(" ↻ Migrating progress file from old tiered system to new format");
console.warn(" Phase checkpoint reset to 0 (will re-scan all 200 diseases)");
console.warn(" Per-class progress (seenUrls, counts) preserved");
raw.phase = 0;
raw.phaseIndex = 0;
} else {
raw.phase ??= 0;
raw.phaseIndex ??= 0;
}
// Ensure each class has the sources field
for (const key of Object.keys(raw.classes)) {
const cp = raw.classes[key] as Partial<ClassProgress>;
cp.sources ??= {
db: { exhausted: false },
duckduckgo: { exhausted: false },
inaturalist: { exhausted: false },
wikimedia: { exhausted: false },
};
// Migrate class-level exhausted to per-source exhausted if needed
if (!cp.sources) {
const classExhausted = cp.exhausted ?? false;
cp.sources = {
db: { exhausted: classExhausted },
duckduckgo: { exhausted: classExhausted },
inaturalist: { exhausted: classExhausted },
wikimedia: { exhausted: classExhausted },
};
}
cp.seenUrls ??= [];
}
return raw as Progress;
@@ -608,7 +817,6 @@ async function collectClassImages(
progress: Progress,
classDir: string,
existingUrls: string[] = [],
fastMode = false, // Skip slow DuckDuckGo, use iNat + Commons only
): Promise<void> {
const cp = getClassProgress(progress, classId);
@@ -664,7 +872,7 @@ async function collectClassImages(
}
// ── Source 1: DuckDuckGo ──────────────────────────────────────────────
if (!fastMode && !sources.duckduckgo.exhausted && allUrls.length < needed) {
if (!sources.duckduckgo.exhausted && allUrls.length < needed) {
for (const query of queries) {
if (allUrls.length >= needed) break;
process.stdout.write(` DDG: "${query.substring(0, 40)}"... `);
@@ -753,7 +961,9 @@ async function collectClassImages(
const pct = Math.round((cp.count / target) * 100);
console.log(
` ${downloaded > 0 ? "✓" : "✗"} Got ${downloaded}/${allUrls.length} (${failed} failed). Total: ${cp.count}/${target} (${pct}%)`,
` ${downloaded > 0 ? "✓" : "✗"} Got ${downloaded}/${
allUrls.length
} (${failed} failed). Total: ${cp.count}/${target} (${pct}%)`,
);
}
@@ -761,21 +971,18 @@ async function collectClassImages(
async function main() {
console.log("=".repeat(60));
console.log("PLANT DISEASE DATASET COLLECTOR — FULL DB");
console.log("PLANT DISEASE DATASET COLLECTOR — TOP 200 COMMON DISEASES");
console.log("=".repeat(60));
// Ensure dataset directory exists before any cache writes
mkdirSync(DATASET_DIR, { recursive: true });
// Load diseases from DB
console.log("\nLoading diseases from database...");
console.log("\nLoading top 200 most common diseases from database...");
const dbDiseases = await loadDiseasesFromDb();
console.log(` ${dbDiseases.length} diseases loaded`);
const coreDiseases = dbDiseases.filter((d) => CORE_PLANTS.has(d.plantId));
const fullDiseases = dbDiseases.filter((d) => !CORE_PLANTS.has(d.plantId));
console.log(` Core plants: ${coreDiseases.length} diseases (target: ${TARGET_CORE})`);
console.log(` Full set: ${fullDiseases.length} diseases (target: ${TARGET_FULL})`);
// Load progress
mkdirSync(DATASET_DIR, { recursive: true });
const progress = loadProgress();
// If all phases complete, exit early
@@ -787,63 +994,57 @@ async function main() {
const startTime = Date.now();
// ── Phase 1: Core set ──────────────────────────────────────────────────
// ── Phase 1: Common diseases (200 images each) ──────────────────────────
console.log("\n" + "─".repeat(60));
console.log("PHASE 1: Core Diseases (100 images each)");
console.log("PHASE 1: Common Diseases (200 images each)");
console.log("─".repeat(60));
const coreStart = progress.phase === 0 ? progress.phaseIndex : 0;
if (coreStart > 0) {
const diseaseStart = progress.phase === 0 ? progress.phaseIndex : 0;
if (diseaseStart > 0) {
console.log(
` Resuming from disease #${coreStart + 1} (${((coreStart / coreDiseases.length) * 100).toFixed(0)}% done)`,
` Resuming from disease #${diseaseStart + 1} (${(
(diseaseStart / dbDiseases.length) *
100
).toFixed(0)}% done)`,
);
}
for (let i = coreStart; i < coreDiseases.length; i++) {
const d = coreDiseases[i];
const classDir = resolve(DATASET_DIR, d.id);
const queries = buildSearchQueries(d);
const existingUrls = d.imageUrl ? [d.imageUrl] : [];
// Process diseases in parallel batches
for (let i = diseaseStart; i < dbDiseases.length; i += DISEASE_CONCURRENCY) {
const batch = dbDiseases.slice(i, i + DISEASE_CONCURRENCY);
const batchNum = Math.floor(i / DISEASE_CONCURRENCY) + 1;
const totalBatches = Math.ceil(dbDiseases.length / DISEASE_CONCURRENCY);
const pct = Math.round((i / dbDiseases.length) * 100);
const pct = Math.round((i / coreDiseases.length) * 100);
console.log(`\n[${i + 1}/${coreDiseases.length}] (${pct}%) ${d.name || d.id} (${d.plantId})`);
console.log(
`\n[Batch ${batchNum}/${totalBatches}] (${pct}%) Processing ${batch.length} diseases in parallel...`,
);
await collectClassImages(d.id, queries, TARGET_CORE, progress, classDir, existingUrls);
// Process all diseases in this batch concurrently
await Promise.all(
batch.map(async (d, batchIdx) => {
const diseaseIdx = i + batchIdx;
const classDir = resolve(DATASET_DIR, d.id);
const queries = buildSearchQueries(d);
const existingUrls = d.imageUrl ? [d.imageUrl] : [];
// Save checkpoint: phase 0, at index i
console.log(` [${diseaseIdx + 1}/${dbDiseases.length}] ${d.name || d.id} (${d.plantId})`);
await collectClassImages(
d.id,
queries,
TARGET_PER_DISEASE,
progress,
classDir,
existingUrls,
);
}),
);
// Save checkpoint: phase 0, at index i + batch.length
progress.phase = 0;
progress.phaseIndex = i + 1;
saveProgress(progress);
}
// ── Phase 2: Full set ──────────────────────────────────────────────────
console.log("\n" + "─".repeat(60));
console.log("PHASE 2: Full Disease Set (10 images each)");
console.log("─".repeat(60));
const fullStart = progress.phase === 1 ? progress.phaseIndex : 0;
if (fullStart > 0) {
console.log(
` Resuming from disease #${fullStart + 1} (${((fullStart / fullDiseases.length) * 100).toFixed(0)}% done)`,
);
}
for (let i = fullStart; i < fullDiseases.length; i++) {
const d = fullDiseases[i];
const classDir = resolve(DATASET_DIR, d.id);
const queries = buildSearchQueries(d);
const existingUrls = d.imageUrl ? [d.imageUrl] : [];
const pct = Math.round((i / fullDiseases.length) * 100);
console.log(`\n[${i + 1}/${fullDiseases.length}] (${pct}%) ${d.id} (${d.plantId})`);
await collectClassImages(d.id, queries, TARGET_FULL, progress, classDir, existingUrls, true);
// Save checkpoint: phase 1, at index i
progress.phase = 1;
progress.phaseIndex = i + 1;
progress.phaseIndex = i + batch.length;
saveProgress(progress);
}