feat(feed): bound feed lifecycle to 30-day window with nonblocking refresh
Persisted feeds keep only episodes from the last 30 days (plus completed downloads); older episodes live in volatile memory and survive refreshes via union merge, with per-feed in-memory caches capped at 500. Refresh batches run at FETCH_CONCURRENCY=4 with per-feed incremental apply (no Promise.all barrier), config.json writes are trailing-edge debounced (250ms, immediate flushPendingSave for unsubscribes), and cold fetch-more refetches abort at FETCH_TIMEOUT_MS. A shared activity store powers a global top-right indicator covering refresh, fetch-more, subscribe, search, and downloads. Also includes the in-flight incremental RSS parsing (chunked with event-loop yields) and refresh spinner work this tree already carried.
This commit is contained in:
@@ -93,6 +93,13 @@ export function updateConfig(patch: Partial<PodTuiConfig>): void {
|
||||
});
|
||||
}
|
||||
|
||||
/** Resolve once every queued config write has flushed. Tests await this to
|
||||
* observe the serialized result of pending saveFeedsToFile/updateConfig
|
||||
* calls before asserting on config.json. */
|
||||
export function whenConfigIdle(): Promise<void> {
|
||||
return writeChain;
|
||||
}
|
||||
|
||||
/** Guards so migration runs exactly once per process. */
|
||||
let migrationDone = false;
|
||||
let migrationPromise: Promise<void> | null = null;
|
||||
|
||||
26
src/utils/episode-merge.ts
Normal file
26
src/utils/episode-merge.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import type { Episode } from "../types/episode"
|
||||
|
||||
/** Sort key for an episode's pubDate — missing/invalid dates sort as NEWEST
|
||||
* (Infinity) so undated episodes float to the top instead of dropping into
|
||||
* the oldest slot. */
|
||||
const ts = (ep: Episode): number => {
|
||||
const t = ep.pubDate?.getTime()
|
||||
return t === undefined || Number.isNaN(t) ? Infinity : t
|
||||
}
|
||||
|
||||
/**
|
||||
* Union of two episode lists keyed by id — on collision the fetched copy
|
||||
* wins (fresh metadata). Result is sorted newest-first by pubDate and capped
|
||||
* at `cap` entries (oldest dropped). Never mutates either input.
|
||||
*/
|
||||
export function mergeEpisodes(
|
||||
existing: Episode[],
|
||||
fetched: Episode[],
|
||||
cap: number,
|
||||
): Episode[] {
|
||||
const byId = new Map<string, Episode>()
|
||||
for (const ep of existing) byId.set(ep.id, ep)
|
||||
for (const ep of fetched) byId.set(ep.id, ep)
|
||||
const sorted = [...byId.values()].sort((a, b) => ts(b) - ts(a))
|
||||
return sorted.slice(0, cap)
|
||||
}
|
||||
@@ -4,9 +4,57 @@
|
||||
*/
|
||||
|
||||
import { loadConfig, updateConfig } from "./config";
|
||||
import { getConfigFilePath } from "./config-dir";
|
||||
import { DownloadStatus } from "../types/episode";
|
||||
import type { Episode } from "../types/episode";
|
||||
import type { Feed } from "../types/feed";
|
||||
import type { PodcastSource } from "../types/source";
|
||||
|
||||
/** Retention window for persisted episodes: older episodes are dropped when
|
||||
* feeds are written to config.json unless they are completed downloads. */
|
||||
export const PERSISTED_WINDOW_DAYS = 30;
|
||||
|
||||
/** True when an episode may be persisted: it is a completed download, or its
|
||||
* pubDate is missing/invalid (fail-safe: never drop an undatable episode),
|
||||
* or it falls inside the retention window. */
|
||||
export function episodeIsPersistable(
|
||||
ep: Episode,
|
||||
downloadedIds: Set<string>,
|
||||
now: Date,
|
||||
): boolean {
|
||||
if (downloadedIds.has(ep.id)) return true;
|
||||
const t = ep.pubDate?.getTime();
|
||||
if (!t || Number.isNaN(t)) return true;
|
||||
return t >= now.getTime() - PERSISTED_WINDOW_DAYS * 24 * 3600 * 1000;
|
||||
}
|
||||
|
||||
/** Episode ids of completed downloads, read from downloads.json. In-flight
|
||||
* downloads are NOT exempted from the retention window — a just-completed
|
||||
* download is re-included by the next save because the in-memory
|
||||
* feed.episodes still holds it. Missing/unreadable/invalid file → empty set. */
|
||||
async function readDownloadedEpisodeIds(): Promise<Set<string>> {
|
||||
try {
|
||||
const file = Bun.file(getConfigFilePath("downloads.json"));
|
||||
if (!(await file.exists())) return new Set();
|
||||
const raw = await file.json();
|
||||
if (!Array.isArray(raw)) return new Set();
|
||||
const ids = new Set<string>();
|
||||
for (const rec of raw) {
|
||||
if (
|
||||
rec &&
|
||||
typeof rec === "object" &&
|
||||
rec.status === DownloadStatus.COMPLETED &&
|
||||
typeof rec.episodeId === "string"
|
||||
) {
|
||||
ids.add(rec.episodeId);
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
/** Deserialize date strings back to Date objects in feed data */
|
||||
function reviveDates(feed: Feed): Feed {
|
||||
return {
|
||||
@@ -23,20 +71,54 @@ function reviveDates(feed: Feed): Feed {
|
||||
};
|
||||
}
|
||||
|
||||
/** Load feeds from config.json */
|
||||
/** Load feeds from config.json, pruning episodes outside the retention
|
||||
* window (completed downloads always kept). When anything was pruned, the
|
||||
* pruned list is rewritten to config.json (startup cleanup for legacy
|
||||
* configs). The read path is awaited so the returned value is deterministic. */
|
||||
export async function loadFeedsFromFile(): Promise<Feed[]> {
|
||||
try {
|
||||
const cfg = await loadConfig();
|
||||
if (!Array.isArray(cfg.feeds)) return [];
|
||||
return cfg.feeds.map(reviveDates);
|
||||
const feeds = cfg.feeds.map(reviveDates);
|
||||
const downloadedIds = await readDownloadedEpisodeIds();
|
||||
const now = new Date();
|
||||
let prunedAny = false;
|
||||
const pruned = feeds.map((f) => {
|
||||
const kept = f.episodes.filter((ep) =>
|
||||
episodeIsPersistable(ep, downloadedIds, now),
|
||||
);
|
||||
if (kept.length !== f.episodes.length) prunedAny = true;
|
||||
return { ...f, episodes: kept };
|
||||
});
|
||||
if (prunedAny) {
|
||||
// Fire-and-forget cleanup rewrite of the legacy config.
|
||||
saveFeedsToFile(pruned);
|
||||
}
|
||||
return pruned;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Save feeds to config.json */
|
||||
/** Save feeds to config.json, pruning episodes outside the retention window
|
||||
* (completed downloads always kept). Fire-and-forget: the prune reads
|
||||
* downloads.json asynchronously, then enqueues the write. On any error the
|
||||
* UNPRUNED feeds are saved instead, so data is never lost. */
|
||||
export function saveFeedsToFile(feeds: Feed[]): void {
|
||||
updateConfig({ feeds });
|
||||
(async () => {
|
||||
try {
|
||||
const downloadedIds = await readDownloadedEpisodeIds();
|
||||
const pruned = feeds.map((f) => ({
|
||||
...f,
|
||||
episodes: f.episodes.filter((ep) =>
|
||||
episodeIsPersistable(ep, downloadedIds, new Date()),
|
||||
),
|
||||
}));
|
||||
updateConfig({ feeds: pruned });
|
||||
} catch {
|
||||
updateConfig({ feeds }); /* never lose data on an error path */
|
||||
}
|
||||
})().catch(() => {});
|
||||
}
|
||||
|
||||
/** Load sources from config.json */
|
||||
|
||||
Reference in New Issue
Block a user