plant-disease-id/scripts/fill-training-dataset.ts

#!/usr/bin/env node
/**
 * fill-training-dataset.ts
 *
 * Scans the existing dataset directory and downloads any missing images
 * to reach the target counts (200 per disease, 400 for healthy).
 *
 * Does NOT re-run prevalence queries — just fills gaps from image sources.
 * Each run scans the directory, reports deficits, then fills them.
 * Interrupt-safe: re-run to pick up where you left off.
 *
 * Parallelism strategy:
 *   - Disease-level: 30 diseases processed concurrently
 *   - Per disease: all 3 DDG queries run in parallel
 *   - Per query: all search pages fetched in parallel
 *   - Per disease: DDG, iNaturalist, and Wikimedia Commons all run concurrently
 *   - A shared DDG token-bucket rate limiter prevents bans
 *
 * Usage: cd apps/web && npx tsx scripts/fill-training-dataset.ts
 */

import "dotenv/config";
import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from "fs";
import { resolve, extname } from "path";

// Load .env.development for DB creds
const envPath = resolve(__dirname, "../.env.development");
try {
  const env = readFileSync(envPath, "utf-8");
  for (const line of env.split("\n")) {
    const trimmed = line.trim();
    if (trimmed && !trimmed.startsWith("#")) {
      const eqIdx = trimmed.indexOf("=");
      if (eqIdx > 0) {
        const key = trimmed.slice(0, eqIdx).trim();
        const val = trimmed.slice(eqIdx + 1).trim();
        if (!process.env[key]) process.env[key] = val;
      }
    }
  }
} catch {}

import { getDb, closeDb } from "@/lib/db/index";
import { diseases } from "@/lib/db/schema";

// ─── Config ─────────────────────────────────────────────────────────────────

const DATASET_DIR = resolve(__dirname, "../data/dataset");
const SEEN_CACHE_FILE = resolve(DATASET_DIR, ".fill-seen-urls.json");

/** Target images per disease */
const TARGET_PER_DISEASE = 200;

/** Target images for the "healthy" class */
const TARGET_HEALTHY = 400;

/**
 * How many diseases to process in parallel.
 * Each disease is I/O-bound (HTTP requests), so high concurrency is safe.
 * The global DDG rate limiter prevents us from overwhelming DuckDuckGo.
 */
const DISEASE_CONCURRENCY = 20;

/**
 * Max DDG requests per second (shared across all concurrent diseases).
 * DuckDuckGo is fairly tolerant, but we still want to be polite.
 * With DISEASE_CONCURRENCY=30, each disease fires 3 parallel queries with
 * parallel pages = 9 parallel DDG requests per disease at peak.
 * The rate limiter serializes this so we don't get banned.
 */
const DDG_RATE_LIMIT_RPS = 2;

/** Max concurrent image downloads per disease */
const CONCURRENT_DOWNLOADS = 2;

/** Minimum image size in bytes to accept */
const MIN_IMAGE_SIZE = 10_000; // 10KB

/** Maximum image size in bytes */
const MAX_IMAGE_SIZE = 10 * 1024 * 1024; // 10MB

/** Allowed file extensions */
const ALLOWED_EXTENSIONS = [".jpg", ".jpeg", ".png", ".webp"];

/** User agent for requests */
const UA =
  "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1";

/** Healthy class directory name */
const HEALTHY_CLASS = "healthy";

/** How often (in diseases processed) to flush the seen-URLs cache to disk */
const SEEN_CACHE_FLUSH_INTERVAL = 20;

/** Max DDG pages to fetch per query.
 *  Each page returns ~100 image results, so 3 pages × 3 queries = ~900 raw URLs
 *  before dedup — more than enough to find 200 unique, valid images. */
const MAX_DDG_PAGES = 3;

/** Healthy source queries limit */
const MAX_HEALTHY_QUERIES = 20;

// ─── Types ──────────────────────────────────────────────────────────────────

interface DuckDuckGoImageResult {
  image: string;
  title: string;
  url: string;
  thumbnail: string;
  height: number;
  width: number;
}

interface DiseaseInfo {
  id: string;
  name: string;
  plantId: string;
  have: number;
  needed: number;
}

interface CollectResult {
  urls: string[];
  exhausted: boolean;
}

// ─── Token-Bucket Rate Limiter ──────────────────────────────────────────────

class TokenBucket {
  private tokens: number;
  private lastRefill: number;
  private readonly capacity: number;
  private readonly refillInterval: number; // ms per token (e.g., 100ms for 10 rps)

  constructor(rps: number) {
    this.capacity = rps;
    this.tokens = rps;
    this.lastRefill = Date.now();
    this.refillInterval = 1000 / rps;
  }

  /** Acquire one token, blocking until one is available. */
  async acquire(): Promise<void> {
    while (true) {
      this.refill();
      if (this.tokens >= 1) {
        this.tokens -= 1;
        return;
      }
      // No tokens — wait for the next one to arrive, then retry
      await sleep(Math.ceil(this.refillInterval));
    }
  }

  private refill(): void {
    const now = Date.now();
    const elapsed = now - this.lastRefill;
    const newTokens = Math.floor(elapsed / this.refillInterval);
    if (newTokens > 0) {
      this.tokens = Math.min(this.capacity, this.tokens + newTokens);
      this.lastRefill = now - (elapsed % this.refillInterval);
    }
  }
}

// Global DDG rate limiter — all concurrent diseases share this
const ddgLimiter = new TokenBucket(DDG_RATE_LIMIT_RPS);

// ─── Helpers ────────────────────────────────────────────────────────────────

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

/** Count actual image files in a directory (matching img_* pattern). */
function countImagesInDir(dir: string): number {
  if (!existsSync(dir)) return 0;
  try {
    const files = readdirSync(dir);
    return files.filter((f) => f.startsWith("img_")).length;
  } catch {
    return 0;
  }
}

// ─── Seen-URLs Cache ──────────────────────────────────────────────────────

/**
 * Load the per-disease seen-URLs cache from disk.
 * This prevents re-fetching the same URLs across runs.
 */
function loadSeenUrlsCache(): Record<string, string[]> {
  if (existsSync(SEEN_CACHE_FILE)) {
    try {
      return JSON.parse(readFileSync(SEEN_CACHE_FILE, "utf-8"));
    } catch {}
  }
  return {};
}

/**
 * Save the seen-URLs cache to disk.
 */
function saveSeenUrlsCache(cache: Record<string, string[]>): void {
  writeFileSync(SEEN_CACHE_FILE, JSON.stringify(cache, null, 2));
}

// ─── DDG VQD Token Cache ──────────────────────────────────────────────────

/**
 * Simple in-memory cache for DDG VQD tokens.
 * Tokens are per-query, but if we've fetched one for a similar query recently,
 * we can skip the initial HTML page fetch.
 */
const vqdCache = new Map<string, { token: string; expiresAt: number }>();

function getCachedVqd(query: string): string | undefined {
  const entry = vqdCache.get(query);
  if (entry && entry.expiresAt > Date.now()) return entry.token;
  vqdCache.delete(query);
  return undefined;
}

function setCachedVqd(query: string, token: string): void {
  // VQD tokens seem to be valid for a few minutes; cache for 5 min
  vqdCache.set(query, { token, expiresAt: Date.now() + 5 * 60 * 1000 });
  // Evict oldest entries if cache grows too large (unlikely but safe)
  if (vqdCache.size > 500) {
    const firstKey = vqdCache.keys().next().value;
    if (firstKey) vqdCache.delete(firstKey);
  }
}

// ─── DuckDuckGo API ─────────────────────────────────────────────────────────

async function getVqdToken(query: string): Promise<string> {
  const cached = getCachedVqd(query);
  if (cached) return cached;

  const url = `https://duckduckgo.com/?q=${encodeURIComponent(query)}&t=h_&iax=images&ia=images`;

  const res = await fetch(url, {
    headers: { "User-Agent": UA, Accept: "text/html" },
    signal: AbortSignal.timeout(15_000),
  });

  if (!res.ok) throw new Error(`Failed to get vqd token: ${res.status}`);

  const html = await res.text();
  const match = html.match(/vqd['"]?\s*[:=]\s*['"]([a-f0-9-]+)['"]/);
  if (!match) throw new Error(`Could not extract vqd token for "${query}"`);

  setCachedVqd(query, match[1]);
  return match[1];
}

async function searchImagesDuckDuckGo(
  query: string,
  vqd: string,
  page: number,
): Promise<DuckDuckGoImageResult[]> {
  // Rate-limit before making the request
  await ddgLimiter.acquire();

  const url = `https://duckduckgo.com/i.js?q=${encodeURIComponent(
    query,
  )}&vqd=${vqd}&o=json&p=${page}&f=,,,`;

  const res = await fetch(url, {
    headers: {
      "User-Agent": UA,
      Accept: "application/json",
      Referer: `https://duckduckgo.com/?q=${encodeURIComponent(query)}&t=h_&iax=images&ia=images`,
    },
    signal: AbortSignal.timeout(15_000),
  });

  if (!res.ok) {
    if (res.status === 429) {
      // Rate limited — wait and retry once
      await sleep(5_000);
      return searchImagesDuckDuckGo(query, vqd, page);
    }
    if (res.status === 403) return [];
    // Don't throw for transient errors — just return empty
    return [];
  }

  const data = (await res.json()) as { results: DuckDuckGoImageResult[] };
  return data.results ?? [];
}

/**
 * Collect images from DDG for a single query.
 * Fetches up to MAX_DDG_PAGES pages in PARALLEL (rate-limited via ddgLimiter).
 */
async function collectFromDdgQuery(
  query: string,
  target: number,
  seenUrls: Set<string>,
): Promise<CollectResult> {
  const results: string[] = [];

  let vqd: string;
  try {
    vqd = await getVqdToken(query);
  } catch (err) {
    console.warn(`    ⚠ DDG token failed: ${err instanceof Error ? err.message : "unknown"}`);
    return { urls: [], exhausted: true };
  }

  // Fetch all pages in parallel
  const pageFetches: Promise<DuckDuckGoImageResult[]>[] = [];
  for (let page = 1; page <= MAX_DDG_PAGES; page++) {
    pageFetches.push(searchImagesDuckDuckGo(query, vqd, page));
  }

  const pageResults = await Promise.allSettled(pageFetches);

  for (const settled of pageResults) {
    if (settled.status !== "fulfilled") continue;
    if (results.length >= target) break;

    for (const r of settled.value) {
      if (results.length >= target) break;
      const imgUrl = r.image || r.url;
      if (!imgUrl || typeof imgUrl !== "string") continue;
      if (seenUrls.has(imgUrl)) continue;
      try {
        new URL(imgUrl);
      } catch {
        continue;
      }
      seenUrls.add(imgUrl);
      results.push(imgUrl);
    }
  }

  return { urls: results.slice(0, target), exhausted: results.length < target };
}

/**
 * Collect images from DDG across ALL queries for a disease.
 * Runs all queries in PARALLEL, then merges deduplicated results.
 */
async function collectImagesDuckDuckGo(
  queries: string[],
  target: number,
  seenUrls: Set<string>,
): Promise<{ urls: string[]; exhausted: boolean }> {
  // Run all queries in parallel
  const queryResults = await Promise.allSettled(
    queries.map((q) => collectFromDdgQuery(q, target, seenUrls)),
  );

  // Merge results — seenUrls already deduplicates across queries
  const merged: string[] = [];
  for (const settled of queryResults) {
    if (settled.status === "fulfilled") {
      merged.push(...settled.value.urls);
      if (merged.length >= target) break;
    }
  }

  return { urls: merged.slice(0, target), exhausted: merged.length < target };
}

// ─── iNaturalist API ───────────────────────────────────────────────────────

async function searchImagesInaturalist(
  query: string,
  target: number,
  seenUrls: Set<string>,
): Promise<CollectResult> {
  const results: string[] = [];
  const perPage = Math.min(target, 200);

  const apiUrl =
    `https://api.inaturalist.org/v1/observations` +
    `?q=${encodeURIComponent(query)}` +
    `&photos_only=true` +
    `&quality_grade=research` +
    `&per_page=${perPage}` +
    `&order_by=observed_on&order=desc`;

  try {
    const res = await fetch(apiUrl, {
      headers: { "User-Agent": UA, Accept: "application/json" },
      signal: AbortSignal.timeout(15_000),
    });
    if (!res.ok) return { urls: [], exhausted: false };

    const data = (await res.json()) as {
      results: Array<{ photos: Array<{ url: string }> }>;
    };

    for (const obs of data.results ?? []) {
      if (results.length >= target) break;
      for (const photo of obs.photos ?? []) {
        if (results.length >= target) break;
        const url = photo.url;
        if (!url || seenUrls.has(url)) continue;
        const fullUrl = url.replace("/medium.", "/original.");
        seenUrls.add(fullUrl);
        results.push(fullUrl);
      }
    }

    return { urls: results, exhausted: results.length < target };
  } catch {
    return { urls: results, exhausted: false };
  }
}

// ─── Wikimedia Commons API ─────────────────────────────────────────────────

async function searchImagesCommons(
  query: string,
  target: number,
  seenUrls: Set<string>,
): Promise<CollectResult> {
  const results: string[] = [];
  let sroffset = 0;

  while (results.length < target) {
    const params = new URLSearchParams({
      action: "query",
      list: "search",
      srsearch: query,
      srnamespace: "6",
      srlimit: "50",
      sroffset: String(sroffset),
      format: "json",
    });

    const url = `https://commons.wikimedia.org/w/api.php?${params}`;

    try {
      const res = await fetch(url, {
        headers: { "User-Agent": UA },
        signal: AbortSignal.timeout(10_000),
      });
      if (!res.ok) break;

      const data = (await res.json()) as {
        query?: { search?: Array<{ title: string }> };
        continue?: { sroffset?: number };
      };

      const hits = data.query?.search ?? [];
      if (hits.length === 0) break;

      for (const hit of hits) {
        if (results.length >= target) break;
        const filename = hit.title.replace(/^File:/, "");
        const imgUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(
          filename,
        )}`;
        if (seenUrls.has(imgUrl)) continue;
        seenUrls.add(imgUrl);
        results.push(imgUrl);
      }

      sroffset = data.continue?.sroffset ?? sroffset + hits.length;
    } catch {
      break;
    }
  }

  return { urls: results, exhausted: results.length < target };
}

// ─── Image Download ─────────────────────────────────────────────────────────

async function downloadImage(url: string, destPath: string): Promise<boolean> {
  try {
    const res = await fetch(url, {
      headers: { "User-Agent": UA, Accept: "image/webp,image/png,image/jpeg,*/*" },
      signal: AbortSignal.timeout(8_000),
    });
    if (!res.ok) return false;

    const contentType = res.headers.get("content-type") || "";
    if (contentType.includes("text/html")) return false;

    const buffer = Buffer.from(await res.arrayBuffer());
    if (buffer.length < MIN_IMAGE_SIZE) return false;
    if (buffer.length > MAX_IMAGE_SIZE) return false;

    let ext = extname(new URL(url).pathname).toLowerCase();
    if (!ALLOWED_EXTENSIONS.includes(ext)) {
      if (contentType.includes("jpeg") || contentType.includes("jpg")) ext = ".jpg";
      else if (contentType.includes("png")) ext = ".png";
      else if (contentType.includes("webp")) ext = ".webp";
      else ext = ".jpg";
    }

    const filePath = destPath.replace(/\.\w+$/, ext);
    writeFileSync(filePath, buffer);
    return true;
  } catch {
    return false;
  }
}

async function downloadBatch(
  urls: string[],
  classDir: string,
  startIndex: number,
): Promise<{ downloaded: number; failed: number; lastIndex: number }> {
  let downloaded = 0;
  let failed = 0;
  let index = startIndex;

  for (let i = 0; i < urls.length; i += CONCURRENT_DOWNLOADS) {
    const chunk = urls.slice(i, i + CONCURRENT_DOWNLOADS);

    const results = await Promise.all(
      chunk.map(async (url) => {
        const paddedIndex = String(index).padStart(4, "0");
        const destPath = resolve(classDir, `img_${paddedIndex}.jpg`);
        const success = await downloadImage(url, destPath);
        return { success, index: index++ };
      }),
    );

    for (const r of results) {
      if (r.success) downloaded++;
      else failed++;
    }
  }

  return { downloaded, failed, lastIndex: index };
}

// ─── Query Building ─────────────────────────────────────────────────────────

function buildSearchQueries(name: string, plant: string): string[] {
  return [`${name} ${plant} leaf disease`, `${plant} ${name} symptoms`, `${name} ${plant}`];
}

function buildHealthyQueries(plant: string): string[] {
  const name = plant.replace(/-/g, " ");
  return [
    `healthy ${name} leaf`,
    `${name} leaf closeup`,
    `healthy ${name} plant`,
    `${name} foliage`,
  ];
}

// ─── Fill Logic ─────────────────────────────────────────────────────────────

/**
 * Try to collect up to `needed` images for a disease by hitting all three
 * sources IN PARALLEL. Returns how many new images were actually downloaded.
 *
 * Sources (DDG with its 3 internal queries, iNat, Commons) all run concurrently.
 * As soon as any source completes, its URLs are downloaded immediately while
 * other sources are still searching (pipeline).
 */
async function fillClass(
  _diseaseId: string,
  queries: string[],
  needed: number,
  classDir: string,
  seenUrls: Set<string>,
): Promise<number> {
  if (needed <= 0) return 0;

  mkdirSync(classDir, { recursive: true });
  const startCount = countImagesInDir(classDir);

  // ── Run all sources in parallel, pipelining downloads ──────────────────
  // Start downloading from each source as soon as it returns results, rather
  // than waiting for all sources to complete. DDG is (by far) the richest
  // source, so its results start saving to disk while iNat and Commons are
  // still searching.
  //
  // Each source gets a DEDICATED index range so there's no race condition
  // writing files. DDG gets [startCount, startCount+199], iNat gets
  // [startCount+200, startCount+399], Commons gets [startCount+400,...].
  // The 4-digit filename supports up to 9999, well beyond our 200 target.

  let totalDownloaded = 0;
  let totalFailed = 0;
  let anySuccess = false;

  const collectAndDownload = async (
    label: string,
    collector: () => Promise<CollectResult>,
    indexOffset: number,
  ): Promise<void> => {
    const result = await collector();
    if (result.urls.length === 0) return;
    console.log(`    ${label}: ${result.urls.length} new URLs`);

    // Each source writes to its own non-overlapping range
    const { downloaded, failed } = await downloadBatch(result.urls, classDir, indexOffset);
    totalDownloaded += downloaded;
    totalFailed += failed;
    if (downloaded > 0) anySuccess = true;
  };

  await Promise.allSettled([
    collectAndDownload("DDG", () => collectImagesDuckDuckGo(queries, needed, seenUrls), startCount),
    collectAndDownload(
      "iNat",
      () => searchImagesInaturalist(queries[0], needed, seenUrls),
      startCount + TARGET_PER_DISEASE,
    ),
    collectAndDownload(
      "Commons",
      () => searchImagesCommons(queries[0], needed, seenUrls),
      startCount + 2 * TARGET_PER_DISEASE,
    ),
  ]);

  if (!anySuccess) {
    console.log(`    ✗ No new images found from any source`);
    return 0;
  }

  const newTotal = countImagesInDir(classDir);
  const gained = newTotal - startCount;
  console.log(
    `    ✓ ${totalDownloaded}/${totalDownloaded + totalFailed} downloaded` +
      ` (${totalFailed} failed, ${gained} new files)`,
  );

  return gained;
}

// ─── Directory Scanner ─────────────────────────────────────────────────────

interface ScanResult {
  /** Disease id → how many images currently on disk */
  diseaseCounts: Map<string, number>;
  /** How many healthy images on disk */
  healthyCount: number;
}

function scanDataset(): ScanResult {
  const diseaseCounts = new Map<string, number>();
  let healthyCount = 0;

  if (!existsSync(DATASET_DIR)) {
    return { diseaseCounts, healthyCount: 0 };
  }

  const entries = readdirSync(DATASET_DIR, { withFileTypes: true });

  for (const entry of entries) {
    if (!entry.isDirectory()) continue;
    if (entry.name.startsWith(".")) continue;

    if (entry.name === HEALTHY_CLASS) {
      healthyCount = countImagesInDir(resolve(DATASET_DIR, entry.name));
    } else {
      const count = countImagesInDir(resolve(DATASET_DIR, entry.name));
      if (count > 0) {
        diseaseCounts.set(entry.name, count);
      }
    }
  }

  return { diseaseCounts, healthyCount };
}

// ─── CLI Flags ──────────────────────────────────────────────────────────────

function parseFlags(): { reverse: boolean } {
  const args = process.argv.slice(2);
  return {
    reverse: args.includes("--reverse") || args.includes("-r"),
  };
}

// ─── Main ───────────────────────────────────────────────────────────────────

async function main() {
  const flags = parseFlags();

  console.log("=".repeat(60));
  console.log("TRAINING DATASET FILL — Parallelized gap-filling download");
  if (flags.reverse) console.log("  (reverse order — processing from lowest deficit first)");
  console.log("=".repeat(60));

  // Ensure dataset directory exists
  mkdirSync(DATASET_DIR, { recursive: true });

  // ── Step 1: Scan what we already have ────────────────────────────────────
  console.log("\nScanning existing dataset...");
  const { diseaseCounts, healthyCount } = scanDataset();
  console.log(`  Found ${diseaseCounts.size} disease directories, ${healthyCount} healthy images`);

  // ── Step 2: Load disease info from DB ────────────────────────────────────
  console.log("\nLoading disease info from database...");
  const db = getDb();

  const allDiseases = await db
    .select({
      id: diseases.id,
      plantId: diseases.plantId,
      name: diseases.name,
    })
    .from(diseases);

  // Build a deduplicated map: disease id → first disease info found
  const diseaseInfo = new Map<string, { name: string; plantId: string }>();
  for (const d of allDiseases) {
    if (!diseaseInfo.has(d.id)) {
      diseaseInfo.set(d.id, { name: d.name, plantId: d.plantId });
    }
  }
  console.log(`  Loaded ${diseaseInfo.size} unique diseases from DB`);

  // ── Step 3: Build deficit list ──────────────────────────────────────────
  const deficits: DiseaseInfo[] = [];

  for (const [id, info] of diseaseInfo) {
    const have = diseaseCounts.get(id) ?? 0;
    const needed = TARGET_PER_DISEASE - have;
    if (needed > 0) {
      deficits.push({ id, name: info.name, plantId: info.plantId, have, needed });
    }
  }

  // Sort by deficit size (largest first) so we prioritize the neediest diseases
  deficits.sort((a, b) => b.needed - a.needed);

  // Reverse order if --reverse/-r flag is set (useful to try a different
  // direction when the front of the queue keeps hitting dead URLs)
  if (flags.reverse) deficits.reverse();

  const healthyDeficit = TARGET_HEALTHY - healthyCount;

  console.log(`\n${"=".repeat(60)}`);
  console.log("DEFICIT REPORT");
  console.log(`${"=".repeat(60)}`);
  console.log(`  Diseases needing images: ${deficits.length}/${diseaseInfo.size}`);
  console.log(`  Total images missing:   ${deficits.reduce((s, d) => s + d.needed, 0)}`);
  console.log(`  Healthy deficit:        ${Math.max(0, healthyDeficit)}`);
  console.log(`  Parallelism:            ${DISEASE_CONCURRENCY} diseases at once`);
  console.log(`  DDG rate limit:         ${DDG_RATE_LIMIT_RPS} req/s (shared)`);
  console.log(
    `  Order:                  ${flags.reverse ? "reverse (--reverse)" : "normal (deficit-first)"}`,
  );
  console.log(`${"=".repeat(60)}`);

  if (deficits.length === 0 && healthyDeficit <= 0) {
    console.log("\n  ✓ Nothing to do — all targets met!\n");
    await closeDb();
    return;
  }

  // ── Step 4: Load seen-URLs cache ────────────────────────────────────────
  const seenUrlsCache = loadSeenUrlsCache();
  let totalDownloaded = 0;
  let totalFailed = 0;
  let diseasesProcessed = 0;
  const startTime = Date.now();

  // ── Step 5: Fill disease deficits ───────────────────────────────────────
  if (deficits.length > 0) {
    console.log("\n" + "─".repeat(60));
    console.log(`FILLING ${deficits.length} DISEASES (target: ${TARGET_PER_DISEASE} each)`);
    console.log("─".repeat(60));

    // Process in parallel batches
    for (let i = 0; i < deficits.length; i += DISEASE_CONCURRENCY) {
      const batch = deficits.slice(i, i + DISEASE_CONCURRENCY);
      const batchNum = Math.floor(i / DISEASE_CONCURRENCY) + 1;
      const totalBatches = Math.ceil(deficits.length / DISEASE_CONCURRENCY);

      console.log(`\n[Batch ${batchNum}/${totalBatches}] Processing ${batch.length} diseases...`);

      // Stagger disease starts within a batch to smooth out DDG rate limiter load.
      // Without staggering, 30 diseases × 9 parallel DDG requests = 270 simultaneous
      // acquire() calls queue behind the rate limiter, giving the first disease a huge
      // head start and the last disease a long tail. Staggering by 200ms each spreads
      // the load evenly, reducing tail latency and improving overall throughput.
      const STAGGER_MS = 200;
      const batchResults = await Promise.allSettled(
        batch.map((d, idx) =>
          (async () => {
            if (idx > 0) await sleep(idx * STAGGER_MS);

            const classDir = resolve(DATASET_DIR, d.id);
            const queries = buildSearchQueries(d.name, d.plantId);
            const seen = new Set<string>(seenUrlsCache[d.id] ?? []);

            console.log(
              `  [${d.id}] have ${d.have}, need ${d.needed} more` + ` (${d.name} / ${d.plantId})`,
            );

            const gained = await fillClass(d.id, queries, d.needed, classDir, seen);

            // Update seen-URLs cache for this disease
            seenUrlsCache[d.id] = Array.from(seen);
            return gained;
          })(),
        ),
      );

      // Aggregate batch results
      for (const result of batchResults) {
        if (result.status === "fulfilled") {
          totalDownloaded += result.value;
        } else {
          console.error(`    ✗ Disease failed: ${result.reason}`);
        }
      }

      diseasesProcessed += batch.length;

      // Flush seen-URLs cache to disk periodically (not after every disease)
      if (
        diseasesProcessed % SEEN_CACHE_FLUSH_INTERVAL < batch.length ||
        i + batch.length >= deficits.length
      ) {
        saveSeenUrlsCache(seenUrlsCache);
      }

      const elapsed = Math.round((Date.now() - startTime) / 1000);
      const rate = diseasesProcessed / Math.max(1, elapsed);
      const remaining = deficits.length - diseasesProcessed;
      const eta = remaining / Math.max(0.01, rate);
      console.log(
        `  [Batch ${batchNum}/${totalBatches}] checkpoint — ` +
          `${totalDownloaded} downloaded, ` +
          `${diseasesProcessed}/${deficits.length} diseases (${rate.toFixed(1)}/s, ` +
          `ETA: ${Math.round(eta)}s)`,
      );
    }
  }

  // ── Step 6: Fill healthy deficit ────────────────────────────────────────
  if (healthyDeficit > 0) {
    console.log("\n" + "─".repeat(60));
    console.log(`FILLING HEALTHY CLASS (target: ${TARGET_HEALTHY})`);
    console.log("─".repeat(60));

    const healthyDir = resolve(DATASET_DIR, HEALTHY_CLASS);
    mkdirSync(healthyDir, { recursive: true });

    // Collect all unique plants from the disease info
    const allPlants = [...new Set(diseaseInfo.values())].map((d) => d.plantId);
    const allHealthyQueries: string[] = [];
    for (const plant of allPlants) {
      allHealthyQueries.push(...buildHealthyQueries(plant));
    }

    const healthySeen = new Set<string>(seenUrlsCache[HEALTHY_CLASS] ?? []);
    const healthyNeeded = TARGET_HEALTHY - countImagesInDir(healthyDir);

    // Run all 3 sources in parallel for the healthy class too
    const [ddgUrls, inatUrls, commonsUrls] = await Promise.allSettled([
      collectImagesDuckDuckGo(
        allHealthyQueries.slice(0, MAX_HEALTHY_QUERIES),
        healthyNeeded,
        healthySeen,
      ),
      searchImagesInaturalist(allHealthyQueries[0], healthyNeeded, healthySeen),
      searchImagesCommons(allHealthyQueries[0], healthyNeeded, healthySeen),
    ]);

    const allUrls: string[] = [];
    for (const settled of [ddgUrls, inatUrls, commonsUrls]) {
      if (settled.status === "fulfilled") {
        allUrls.push(...settled.value.urls);
      }
    }

    if (allUrls.length > 0) {
      console.log(`\n  Downloading ${allUrls.length} healthy images...`);
      const startIdx = countImagesInDir(healthyDir);
      const { downloaded, failed } = await downloadBatch(allUrls, healthyDir, startIdx);

      const newTotal = countImagesInDir(healthyDir);
      const gained = newTotal - healthyCount;
      totalDownloaded += gained;
      totalFailed += failed;

      console.log(
        `  ${downloaded > 0 ? "✓" : "✗"} Got ${downloaded} images.` +
          ` Total healthy: ${newTotal}/${TARGET_HEALTHY} (${gained} new)`,
      );
    } else {
      console.log(`\n  ✗ No healthy images found`);
    }

    // Update seen-URLs cache
    seenUrlsCache[HEALTHY_CLASS] = Array.from(healthySeen);
    saveSeenUrlsCache(seenUrlsCache);
  }

  // ── Summary ──────────────────────────────────────────────────────────────
  const elapsed = Math.round((Date.now() - startTime) / 1000);
  const mins = Math.floor(elapsed / 60);
  const hrs = Math.floor(mins / 60);

  // Final scan
  const finalScan = scanDataset();
  const totalHave = [...finalScan.diseaseCounts.values()].reduce((s, c) => s + c, 0);
  const atTarget = [...finalScan.diseaseCounts.values()].filter(
    (c) => c >= TARGET_PER_DISEASE,
  ).length;

  console.log("\n" + "=".repeat(60));
  console.log("  ✅ FILL COMPLETE");
  console.log("=".repeat(60));
  console.log(`  Time:              ${hrs}h ${mins % 60}m`);
  console.log(`  Diseases at target: ${atTarget}/${diseaseInfo.size}`);
  console.log(`  Total images:       ${totalHave}`);
  console.log(`  Healthy images:     ${finalScan.healthyCount}/${TARGET_HEALTHY}`);
  console.log(`  New downloads:      ${totalDownloaded}`);
  console.log(`  Dataset dir:        ${DATASET_DIR}/`);

  await closeDb();
  console.log("=".repeat(60));
}

main().catch((err) => {
  console.error("\nFatal error:", `\n${err}`);
  process.exit(1);
});