feat(feed): bound feed lifecycle to 30-day window with nonblocking refresh
Persisted feeds keep only episodes from the last 30 days (plus completed downloads); older episodes live in volatile memory and survive refreshes via union merge, with per-feed in-memory caches capped at 500. Refresh batches run at FETCH_CONCURRENCY=4 with per-feed incremental apply (no Promise.all barrier), config.json writes are trailing-edge debounced (250ms, immediate flushPendingSave for unsubscribes), and cold fetch-more refetches abort at FETCH_TIMEOUT_MS. A shared activity store powers a global top-right indicator covering refresh, fetch-more, subscribe, search, and downloads. Also includes the in-flight incremental RSS parsing (chunked with event-loop yields) and refresh spinner work this tree already carried.
This commit is contained in:
@@ -74,6 +74,72 @@ const parseEpisodeType = (raw: string): EpisodeType | undefined => {
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** Extract the `<item>` blocks from an RSS document (the sync part of
|
||||
* parsing is bounded to this single regex pass). Exported so the feed store
|
||||
* can parse episodes incrementally without re-deriving item boundaries. */
|
||||
export const getRSSItems = (xml: string): string[] => {
|
||||
const channel = xml.match(/<channel[\s\S]*?<\/channel>/i)?.[0] ?? xml
|
||||
return channel.match(/<item[\s\S]*?<\/item>/gi) ?? []
|
||||
}
|
||||
|
||||
/** Parse a single `<item>` into an Episode. Exported so the feed store can
|
||||
* parse large feeds in bounded chunks (yielding to the event loop between
|
||||
* chunks) instead of one synchronous block. */
|
||||
export const parseRSSItem = (item: string, feedUrl: string, index: number): Episode => {
|
||||
const epTitle = cleanField(getTagValue(item, "title")) || `Episode ${index + 1}`
|
||||
const epDescription = cleanField(getTagValue(item, "description"))
|
||||
const pubDate = new Date(getTagValue(item, "pubDate") || Date.now())
|
||||
|
||||
// Audio URL + file size + MIME type from <enclosure>
|
||||
const enclosure = item.match(/<enclosure[^>]*url=["']([^"']+)["'][^>]*>/i)
|
||||
const audioUrl = enclosure?.[1] ?? ""
|
||||
const fileSizeStr = getAttr(item, "enclosure", "length")
|
||||
const fileSize = fileSizeStr ? parseInt(fileSizeStr, 10) : undefined
|
||||
const mimeType = getAttr(item, "enclosure", "type") || undefined
|
||||
|
||||
// Duration from <itunes:duration>
|
||||
const durationRaw = getTagValue(item, "itunes:duration")
|
||||
const duration = parseDuration(durationRaw)
|
||||
|
||||
// Episode & season numbers
|
||||
const episodeNumRaw = getTagValue(item, "itunes:episode")
|
||||
const episodeNumber = episodeNumRaw ? parseInt(episodeNumRaw, 10) : undefined
|
||||
const seasonNumRaw = getTagValue(item, "itunes:season")
|
||||
const seasonNumber = seasonNumRaw ? parseInt(seasonNumRaw, 10) : undefined
|
||||
|
||||
// Episode type & explicit
|
||||
const episodeType = parseEpisodeType(getTagValue(item, "itunes:episodeType"))
|
||||
const explicitRaw = getTagValue(item, "itunes:explicit").toLowerCase()
|
||||
const explicit = explicitRaw === "yes" || explicitRaw === "true" ? true : undefined
|
||||
|
||||
// Episode image (itunes:image has href attribute)
|
||||
const imageUrl = getAttr(item, "itunes:image", "href") || undefined
|
||||
|
||||
const ep: Episode = {
|
||||
id: `${feedUrl}#${index}`,
|
||||
podcastId: feedUrl,
|
||||
title: epTitle,
|
||||
description: epDescription,
|
||||
audioUrl,
|
||||
duration,
|
||||
pubDate,
|
||||
}
|
||||
|
||||
// Only set optional fields if present
|
||||
if (episodeNumber !== undefined && !isNaN(episodeNumber)) ep.episodeNumber = episodeNumber
|
||||
if (seasonNumber !== undefined && !isNaN(seasonNumber)) ep.seasonNumber = seasonNumber
|
||||
if (episodeType) ep.episodeType = episodeType
|
||||
if (explicit !== undefined) ep.explicit = explicit
|
||||
if (imageUrl) ep.imageUrl = imageUrl
|
||||
if (fileSize !== undefined && !isNaN(fileSize) && fileSize > 0) ep.fileSize = fileSize
|
||||
if (mimeType) ep.mimeType = mimeType
|
||||
|
||||
return ep
|
||||
}
|
||||
|
||||
/** Parse a full RSS document (channel metadata + all episodes). The sync
|
||||
* whole-feed variant — callers that parse potentially huge feeds on a UI
|
||||
* thread should prefer the store's chunked incremental parse instead. */
|
||||
export const parseRSSFeed = (xml: string, feedUrl: string): Podcast & { episodes: Episode[] } => {
|
||||
const channel = xml.match(/<channel[\s\S]*?<\/channel>/i)?.[0] ?? xml
|
||||
const title = cleanField(getTagValue(channel, "title")) || "Untitled Podcast"
|
||||
@@ -81,58 +147,8 @@ export const parseRSSFeed = (xml: string, feedUrl: string): Podcast & { episodes
|
||||
const author = decodeEntities(getTagValue(channel, "itunes:author"))
|
||||
const lastUpdated = new Date()
|
||||
|
||||
const items = channel.match(/<item[\s\S]*?<\/item>/gi) ?? []
|
||||
const episodes = items.map((item, index) => {
|
||||
const epTitle = cleanField(getTagValue(item, "title")) || `Episode ${index + 1}`
|
||||
const epDescription = cleanField(getTagValue(item, "description"))
|
||||
const pubDate = new Date(getTagValue(item, "pubDate") || Date.now())
|
||||
|
||||
// Audio URL + file size + MIME type from <enclosure>
|
||||
const enclosure = item.match(/<enclosure[^>]*url=["']([^"']+)["'][^>]*>/i)
|
||||
const audioUrl = enclosure?.[1] ?? ""
|
||||
const fileSizeStr = getAttr(item, "enclosure", "length")
|
||||
const fileSize = fileSizeStr ? parseInt(fileSizeStr, 10) : undefined
|
||||
const mimeType = getAttr(item, "enclosure", "type") || undefined
|
||||
|
||||
// Duration from <itunes:duration>
|
||||
const durationRaw = getTagValue(item, "itunes:duration")
|
||||
const duration = parseDuration(durationRaw)
|
||||
|
||||
// Episode & season numbers
|
||||
const episodeNumRaw = getTagValue(item, "itunes:episode")
|
||||
const episodeNumber = episodeNumRaw ? parseInt(episodeNumRaw, 10) : undefined
|
||||
const seasonNumRaw = getTagValue(item, "itunes:season")
|
||||
const seasonNumber = seasonNumRaw ? parseInt(seasonNumRaw, 10) : undefined
|
||||
|
||||
// Episode type & explicit
|
||||
const episodeType = parseEpisodeType(getTagValue(item, "itunes:episodeType"))
|
||||
const explicitRaw = getTagValue(item, "itunes:explicit").toLowerCase()
|
||||
const explicit = explicitRaw === "yes" || explicitRaw === "true" ? true : undefined
|
||||
|
||||
// Episode image (itunes:image has href attribute)
|
||||
const imageUrl = getAttr(item, "itunes:image", "href") || undefined
|
||||
|
||||
const ep: Episode = {
|
||||
id: `${feedUrl}#${index}`,
|
||||
podcastId: feedUrl,
|
||||
title: epTitle,
|
||||
description: epDescription,
|
||||
audioUrl,
|
||||
duration,
|
||||
pubDate,
|
||||
}
|
||||
|
||||
// Only set optional fields if present
|
||||
if (episodeNumber !== undefined && !isNaN(episodeNumber)) ep.episodeNumber = episodeNumber
|
||||
if (seasonNumber !== undefined && !isNaN(seasonNumber)) ep.seasonNumber = seasonNumber
|
||||
if (episodeType) ep.episodeType = episodeType
|
||||
if (explicit !== undefined) ep.explicit = explicit
|
||||
if (imageUrl) ep.imageUrl = imageUrl
|
||||
if (fileSize !== undefined && !isNaN(fileSize) && fileSize > 0) ep.fileSize = fileSize
|
||||
if (mimeType) ep.mimeType = mimeType
|
||||
|
||||
return ep
|
||||
})
|
||||
const items = getRSSItems(xml)
|
||||
const episodes = items.map((item, index) => parseRSSItem(item, feedUrl, index))
|
||||
|
||||
return {
|
||||
id: feedUrl,
|
||||
|
||||
53
src/components/GlobalActivityIndicator.tsx
Normal file
53
src/components/GlobalActivityIndicator.tsx
Normal file
@@ -0,0 +1,53 @@
|
||||
import { Show } from "solid-js";
|
||||
import { useFeedStore } from "@/stores/feed";
|
||||
import { useSearchStore } from "@/stores/search";
|
||||
import { useDownloadStore } from "@/stores/download";
|
||||
import { useActivityStore } from "@/stores/activity";
|
||||
import { LoadingIndicator } from "@/components/LoadingIndicator";
|
||||
|
||||
/**
|
||||
* GlobalActivityIndicator — one global top-right signal that ANY feed
|
||||
* refresh, fetch-more, subscribe fetch, search, or download is in flight.
|
||||
* Per-page spinners are unchanged; this overlays the content row and status
|
||||
* bar as a single app-wide "something is happening" indicator.
|
||||
*/
|
||||
export function GlobalActivityIndicator() {
|
||||
const feedStore = useFeedStore();
|
||||
const searchStore = useSearchStore();
|
||||
const downloadStore = useDownloadStore();
|
||||
const activity = useActivityStore();
|
||||
|
||||
/** True while any tracked activity is in flight */
|
||||
const isActive = () =>
|
||||
feedStore.isLoadingFeeds() ||
|
||||
feedStore.isLoadingMore() ||
|
||||
searchStore.isSearching() ||
|
||||
downloadStore.getActiveCount() + downloadStore.getQueue().length > 0 ||
|
||||
activity.isActive();
|
||||
|
||||
/** Label priority: downloads in flight > latest tracked activity >
|
||||
* generic loading (only reachable when an isLoading/isSearching flag is
|
||||
* on but nothing else is). */
|
||||
const label = () => {
|
||||
const activeCount = downloadStore.getActiveCount();
|
||||
const queueLength = downloadStore.getQueue().length;
|
||||
if (activeCount + queueLength > 0) {
|
||||
return `Downloading ${activeCount}${
|
||||
queueLength > 0 ? ` +${queueLength} queued` : ""
|
||||
}`;
|
||||
}
|
||||
if (activity.isActive()) {
|
||||
const latest = activity.labels().at(-1);
|
||||
return `${latest ?? "Loading"}…`;
|
||||
}
|
||||
return "Loading…";
|
||||
};
|
||||
|
||||
return (
|
||||
<Show when={isActive()}>
|
||||
<box position="absolute" top={0} right={0} paddingRight={1}>
|
||||
<LoadingIndicator label={label()} />
|
||||
</box>
|
||||
</Show>
|
||||
);
|
||||
}
|
||||
@@ -63,6 +63,9 @@ export type PaneRowProps = {
|
||||
/** Number of visible columns. `3` (default) = parent|current|preview;
|
||||
* `2` = parent|current (preview omitted, current grows to fill). */
|
||||
panes?: 2 | 3;
|
||||
/** Which sides of the current column's border render. Defaults to
|
||||
* `["left", "right"]` (the standard focused-list frame). */
|
||||
currentBorder?: boolean | BorderSides[];
|
||||
};
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
@@ -181,6 +184,9 @@ export function PaneRow(props: PaneRowProps) {
|
||||
? PANE_RATIO.current + PANE_RATIO.preview
|
||||
: PANE_RATIO.current,
|
||||
);
|
||||
const currentBorder = createMemo<boolean | BorderSides[]>(
|
||||
() => props.currentBorder ?? ["left", "right"],
|
||||
);
|
||||
|
||||
return (
|
||||
<box flexDirection="row" flexGrow={1} width="100%" height="100%">
|
||||
@@ -197,7 +203,7 @@ export function PaneRow(props: PaneRowProps) {
|
||||
grow={currentGrow()}
|
||||
label={() => ""}
|
||||
content={currentContent}
|
||||
border={["left", "right"]}
|
||||
border={currentBorder()}
|
||||
scrollFocused={() => focused()}
|
||||
/>
|
||||
{/* ── preview (30%) — hovered-item detail; no border, no header ────── */}
|
||||
|
||||
@@ -27,6 +27,7 @@ import { TABS } from "@/utils/navigation";
|
||||
import { createDispatcher } from "@/utils/dispatch";
|
||||
import { TabListPane } from "@/components/TabPanel";
|
||||
import { PaneRow } from "@/components/PaneRow";
|
||||
import { GlobalActivityIndicator } from "@/components/GlobalActivityIndicator";
|
||||
|
||||
export function Shell() {
|
||||
const theme = useTheme();
|
||||
@@ -396,6 +397,8 @@ export function Shell() {
|
||||
theme={t as any}
|
||||
/>
|
||||
</Show>
|
||||
{/* ── Global activity indicator (top-right overlay) ─────────────────────── */}
|
||||
<GlobalActivityIndicator />
|
||||
</box>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -236,7 +236,7 @@ function FeedPage() {
|
||||
<Show
|
||||
when={episodes().length > 0}
|
||||
fallback={
|
||||
<box padding={1}>
|
||||
<box padding={1} alignItems="center">
|
||||
<Show
|
||||
when={feedStore.isLoadingFeeds()}
|
||||
fallback={
|
||||
@@ -355,7 +355,7 @@ function FeedPage() {
|
||||
</box>
|
||||
</Show>
|
||||
<Show when={feedStore.isLoadingFeeds()}>
|
||||
<box paddingLeft={2} paddingTop={1}>
|
||||
<box alignItems="center" paddingTop={1}>
|
||||
<LoadingIndicator label="Refreshing…" />
|
||||
</box>
|
||||
</Show>
|
||||
|
||||
@@ -133,6 +133,7 @@ export function PlayerPage() {
|
||||
currentLabel="Player"
|
||||
panes={2}
|
||||
focused={isActive}
|
||||
currentBorder={["left"]}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
75
src/stores/activity.ts
Normal file
75
src/stores/activity.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Activity store for PodTUI
|
||||
*
|
||||
* Shared leak-proof activity counter: any store can surface "something is
|
||||
* loading/downloading" to the global top-right indicator. beginActivity
|
||||
* returns an end token that removes exactly THAT instance, so concurrent
|
||||
* overlapping activities compose correctly; prefer track() so callers
|
||||
* cannot strand the counter.
|
||||
*/
|
||||
|
||||
import { createSignal } from "solid-js";
|
||||
|
||||
/** Create activity store */
|
||||
function createActivityStore() {
|
||||
const [count, setCount] = createSignal(0);
|
||||
const [labels, setLabels] = createSignal<string[]>([]);
|
||||
|
||||
/** Begin a tracked activity and return its end function. Every begin
|
||||
* MUST be paired with exactly one call of the returned end (via the
|
||||
* token); prefer track() so the pairing is automatic. Duplicate labels
|
||||
* are allowed — each end removes exactly one instance (found by
|
||||
* indexOf). */
|
||||
const beginActivity = (label: string): (() => void) => {
|
||||
setLabels((prev) => [...prev, label]);
|
||||
setCount((c) => c + 1);
|
||||
let ended = false;
|
||||
return () => {
|
||||
if (ended) return;
|
||||
ended = true;
|
||||
setLabels((prev) => {
|
||||
const idx = prev.indexOf(label);
|
||||
if (idx === -1) return prev;
|
||||
const next = [...prev];
|
||||
next.splice(idx, 1);
|
||||
return next;
|
||||
});
|
||||
setCount((c) => Math.max(0, c - 1));
|
||||
};
|
||||
};
|
||||
|
||||
/** Track a promise: begin an activity, auto-end when it settles, and
|
||||
* re-throw on rejection so the caller's error handling is untouched. */
|
||||
const track = async <T,>(p: Promise<T>, label: string): Promise<T> => {
|
||||
const end = beginActivity(label);
|
||||
try {
|
||||
return await p;
|
||||
} finally {
|
||||
end();
|
||||
}
|
||||
};
|
||||
|
||||
/** True while at least one activity is in flight */
|
||||
const isActive = (): boolean => count() > 0;
|
||||
|
||||
return {
|
||||
// State
|
||||
count,
|
||||
labels,
|
||||
// Actions
|
||||
beginActivity,
|
||||
track,
|
||||
// Getters
|
||||
isActive,
|
||||
};
|
||||
}
|
||||
|
||||
/** Singleton activity store */
|
||||
let activityStoreInstance: ReturnType<typeof createActivityStore> | null = null;
|
||||
|
||||
export function useActivityStore() {
|
||||
if (!activityStoreInstance) {
|
||||
activityStoreInstance = createActivityStore();
|
||||
}
|
||||
return activityStoreInstance;
|
||||
}
|
||||
@@ -10,15 +10,17 @@ import type { Podcast } from "../types/podcast";
|
||||
import type { Episode } from "../types/episode";
|
||||
import type { PodcastSource } from "../types/source";
|
||||
import { DEFAULT_SOURCES } from "../types/source";
|
||||
import { parseRSSFeed } from "../api/rss-parser";
|
||||
import { getRSSItems, parseRSSItem } from "../api/rss-parser";
|
||||
import { resolveItunesFeedUrl } from "../utils/itunes-feed-resolver";
|
||||
import { savePodcastIndexCredentials } from "../utils/source-credentials";
|
||||
import { mergeEpisodes } from "../utils/episode-merge";
|
||||
import {
|
||||
loadFeedsFromFile,
|
||||
saveFeedsToFile,
|
||||
loadSourcesFromFile,
|
||||
saveSourcesToFile,
|
||||
} from "../utils/feeds-persistence";
|
||||
import { useActivityStore } from "./activity";
|
||||
import { useDownloadStore } from "./download";
|
||||
import { useAppStore } from "./app";
|
||||
import { DownloadStatus } from "../types/episode";
|
||||
@@ -29,13 +31,65 @@ const MAX_EPISODES_REFRESH = 50;
|
||||
/** Max episodes to fetch on initial subscribe */
|
||||
const MAX_EPISODES_SUBSCRIBE = 20;
|
||||
|
||||
/** Per-feed bound on both the cached parse results and the merged in-memory
|
||||
* window; 500 covers years of a weekly show's history while capping a
|
||||
* 20-subscription install at 10k episodes. */
|
||||
export const MAX_EPISODES_IN_MEMORY = 500;
|
||||
|
||||
/** Per-feed fetch timeout — a hung feed must not stall a refresh batch or
|
||||
* the background refresh loop. */
|
||||
const FETCH_TIMEOUT_MS = 20_000;
|
||||
|
||||
/** Bounds simultaneous RSS requests during a refresh batch — a hung feed
|
||||
* burns at most one slot for FETCH_TIMEOUT_MS instead of pinning the whole
|
||||
* batch. */
|
||||
const FETCH_CONCURRENCY = 4;
|
||||
|
||||
/** Default minutes between automatic background feed refreshes. */
|
||||
const DEFAULT_REFRESH_INTERVAL_MINUTES = 30;
|
||||
|
||||
/** Max episodes parsed per chunk before yielding to the event loop — bounds
|
||||
* the synchronous regex work per frame so one huge feed (or a batch of
|
||||
* feeds) can't stall the renderer. */
|
||||
const PARSE_CHUNK_SIZE = 25;
|
||||
|
||||
/** Yield to the event loop (task queue) so the renderer can paint between
|
||||
* parse chunks. MessageChannel instead of setTimeout/setImmediate because
|
||||
* bun:test fake timers trap those (feed-refresh/pagination tests run under
|
||||
* vi.useFakeTimers and await refreshes, so a trapped yield would deadlock
|
||||
* them); MessageChannel posts are real task-queue turns that fire in both
|
||||
* environments. */
|
||||
const yieldToUI = (): Promise<void> =>
|
||||
new Promise((resolve) => {
|
||||
const { port1, port2 } = new MessageChannel();
|
||||
port1.onmessage = () => {
|
||||
port1.close();
|
||||
port2.close();
|
||||
resolve();
|
||||
};
|
||||
port2.postMessage(null);
|
||||
});
|
||||
|
||||
/** Parse all episodes from feed XML in bounded chunks, yielding to the event
|
||||
* loop between chunks. The whole-feed sync `parseRSSFeed` would otherwise
|
||||
* block the UI thread for the combined parse time of every feed in a
|
||||
* refresh batch. */
|
||||
const parseEpisodesIncremental = async (
|
||||
xml: string,
|
||||
feedUrl: string,
|
||||
): Promise<Episode[]> => {
|
||||
const items = getRSSItems(xml);
|
||||
const episodes: Episode[] = new Array(items.length);
|
||||
for (let start = 0; start < items.length; start += PARSE_CHUNK_SIZE) {
|
||||
const end = Math.min(start + PARSE_CHUNK_SIZE, items.length);
|
||||
for (let i = start; i < end; i++) {
|
||||
episodes[i] = parseRSSItem(items[i], feedUrl, i);
|
||||
}
|
||||
if (end < items.length) await yieldToUI();
|
||||
}
|
||||
return episodes;
|
||||
};
|
||||
|
||||
/** Cache of all parsed episodes per feed (feedId -> Episode[]) */
|
||||
const fullEpisodeCache = new Map<string, Episode[]>();
|
||||
|
||||
@@ -96,15 +150,42 @@ async function migratePlaintextCredentials(
|
||||
return changed ? migrated : sources;
|
||||
}
|
||||
|
||||
/** True when two episode lists hold the same episodes (id-set equality,
|
||||
* order-insensitive). Refreshes compare fetched content against this so an
|
||||
* unchanged feed keeps its `lastUpdated` — and therefore its place in the
|
||||
* "updated" sort — instead of reordering the list on every background
|
||||
* refresh. */
|
||||
function sameEpisodes(a: Episode[], b: Episode[]): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
const ids = new Set(a.map((e) => e.id));
|
||||
return b.every((e) => ids.has(e.id));
|
||||
/** True when the freshly fetched window matches the corresponding PREFIX of
|
||||
* the existing episode list (id-set equality, order-insensitive). With
|
||||
* union semantics the merged list legitimately contains episodes BEYOND the
|
||||
* fetched window, so unchanged-detection must compare the fetched window
|
||||
* against the existing list's prefix — comparing full lists would bump
|
||||
* `lastUpdated` on every refresh. */
|
||||
function sameRefreshWindow(existing: Episode[], fetched: Episode[]): boolean {
|
||||
if (fetched.length === 0) return true;
|
||||
const prefix = existing.slice(0, fetched.length);
|
||||
const ids = new Set(prefix.map((e) => e.id));
|
||||
return fetched.every((e) => ids.has(e.id));
|
||||
}
|
||||
|
||||
/** Run `fn` over every item with at most `limit` executions in flight — a
|
||||
* classic worker pool. Workers pull indexes from a shared counter, so the
|
||||
* first `limit` calls start immediately and each completion frees its slot
|
||||
* for the next item; results are assembled in INPUT order regardless of
|
||||
* completion order. A hung `fn` holds at most one slot. */
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results = new Array<R>(items.length);
|
||||
let nextIndex = 0;
|
||||
const workers = Array.from(
|
||||
{ length: Math.min(limit, items.length) },
|
||||
async () => {
|
||||
let i: number;
|
||||
while ((i = nextIndex++) < items.length) {
|
||||
results[i] = await fn(items[i]);
|
||||
}
|
||||
},
|
||||
);
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Create feed store */
|
||||
@@ -122,6 +203,39 @@ function createFeedStore() {
|
||||
const [isLoadingMore, setIsLoadingMore] = createSignal(false);
|
||||
const [isLoadingFeeds, setIsLoadingFeeds] = createSignal(false);
|
||||
|
||||
// ── Debounced persistence ───────────────────────────────────────────────
|
||||
/** Trailing-edge debounce window for config.json writes. */
|
||||
const SAVE_DEBOUNCE_MS = 250;
|
||||
/** True when a save is scheduled but has not flushed yet. */
|
||||
let savePending = false;
|
||||
let pendingSaveTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
/** Schedule a config.json write (trailing edge) — rapid state changes
|
||||
* (a refresh batch landing feed-by-feed, pin toggles, load-more pages)
|
||||
* collapse into one final write instead of one file rewrite per step. */
|
||||
const scheduleSaveFeeds = (): void => {
|
||||
savePending = true;
|
||||
if (pendingSaveTimer) clearTimeout(pendingSaveTimer);
|
||||
pendingSaveTimer = setTimeout(() => {
|
||||
pendingSaveTimer = null;
|
||||
flushPendingSave();
|
||||
}, SAVE_DEBOUNCE_MS);
|
||||
};
|
||||
|
||||
/** Persist immediately when anything is dirty; exported for tests and
|
||||
* quit hooks. Cancels a pending debounced save — the state it would
|
||||
* have written is already reflected in feeds(), so writing now is
|
||||
* strictly more current. */
|
||||
const flushPendingSave = (): void => {
|
||||
if (pendingSaveTimer) {
|
||||
clearTimeout(pendingSaveTimer);
|
||||
pendingSaveTimer = null;
|
||||
}
|
||||
if (!savePending) return;
|
||||
savePending = false;
|
||||
saveFeeds(feeds());
|
||||
};
|
||||
|
||||
/** Get filtered and sorted feeds */
|
||||
const getFilteredFeeds = (): Feed[] => {
|
||||
let result = [...feeds()];
|
||||
@@ -230,12 +344,13 @@ function createFeedStore() {
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
const xml = await response.text();
|
||||
const parsed = parseRSSFeed(xml, feedUrl);
|
||||
const allEpisodes = sortEpisodesReverseChronological(parsed.episodes);
|
||||
const allEpisodes = sortEpisodesReverseChronological(
|
||||
await parseEpisodesIncremental(xml, feedUrl),
|
||||
);
|
||||
|
||||
// Cache all parsed episodes for pagination
|
||||
if (feedId) {
|
||||
fullEpisodeCache.set(feedId, allEpisodes);
|
||||
fullEpisodeCache.set(feedId, allEpisodes.slice(0, MAX_EPISODES_IN_MEMORY));
|
||||
episodeLoadCount.set(feedId, Math.min(limit, allEpisodes.length));
|
||||
}
|
||||
|
||||
@@ -256,45 +371,51 @@ function createFeedStore() {
|
||||
sourceId: string,
|
||||
visibility: FeedVisibility = FeedVisibility.PUBLIC,
|
||||
): Promise<Feed | null> => {
|
||||
// A directory stub (e.g. a show delisted from Apple Podcasts) has no
|
||||
// feed URL; resolve the real feed from its directory page before
|
||||
// subscribing. Refuse when it can't be resolved rather than adding a
|
||||
// broken feed.
|
||||
if (!podcast.feedUrl) {
|
||||
if (!podcast.directoryUrl) return null;
|
||||
const resolved = await resolveItunesFeedUrl(podcast.directoryUrl);
|
||||
if (!resolved) return null;
|
||||
podcast = { ...podcast, feedUrl: resolved, directoryUrl: undefined };
|
||||
}
|
||||
const activity = useActivityStore();
|
||||
// The "Subscribing" label covers the directory-resolve + subscribe
|
||||
// fetch stretch — the gaps no existing signal (isLoadingFeeds,
|
||||
// per-pane spinners) covers.
|
||||
return activity.track((async () => {
|
||||
// A directory stub (e.g. a show delisted from Apple Podcasts) has no
|
||||
// feed URL; resolve the real feed from its directory page before
|
||||
// subscribing. Refuse when it can't be resolved rather than adding a
|
||||
// broken feed.
|
||||
if (!podcast.feedUrl) {
|
||||
if (!podcast.directoryUrl) return null;
|
||||
const resolved = await resolveItunesFeedUrl(podcast.directoryUrl);
|
||||
if (!resolved) return null;
|
||||
podcast = { ...podcast, feedUrl: resolved, directoryUrl: undefined };
|
||||
}
|
||||
|
||||
// Guard: don't add a feed we already have (matched by feedUrl)
|
||||
if (hasFeedByUrl(podcast.feedUrl)) {
|
||||
return feeds().find((f) => f.podcast.feedUrl === podcast.feedUrl) ?? null;
|
||||
}
|
||||
// Guard: don't add a feed we already have (matched by feedUrl)
|
||||
if (hasFeedByUrl(podcast.feedUrl)) {
|
||||
return feeds().find((f) => f.podcast.feedUrl === podcast.feedUrl) ?? null;
|
||||
}
|
||||
|
||||
const feedId = crypto.randomUUID();
|
||||
const episodes = await fetchEpisodes(
|
||||
podcast.feedUrl,
|
||||
MAX_EPISODES_SUBSCRIBE,
|
||||
feedId,
|
||||
);
|
||||
const newFeed: Feed = {
|
||||
id: feedId,
|
||||
podcast,
|
||||
episodes: episodes ?? [],
|
||||
visibility,
|
||||
sourceId,
|
||||
lastUpdated: new Date(),
|
||||
isPinned: false,
|
||||
};
|
||||
setFeeds((prev) => {
|
||||
const updated = [...prev, newFeed];
|
||||
saveFeeds(updated);
|
||||
return updated;
|
||||
});
|
||||
// Global auto-download: newly subscribed shows join the next pass.
|
||||
runAutoDownload();
|
||||
return newFeed;
|
||||
const feedId = crypto.randomUUID();
|
||||
const episodes = await fetchEpisodes(
|
||||
podcast.feedUrl,
|
||||
MAX_EPISODES_SUBSCRIBE,
|
||||
feedId,
|
||||
);
|
||||
const newFeed: Feed = {
|
||||
id: feedId,
|
||||
podcast,
|
||||
episodes: episodes ?? [],
|
||||
visibility,
|
||||
sourceId,
|
||||
lastUpdated: new Date(),
|
||||
isPinned: false,
|
||||
};
|
||||
setFeeds((prev) => {
|
||||
const updated = [...prev, newFeed];
|
||||
scheduleSaveFeeds();
|
||||
return updated;
|
||||
});
|
||||
// Global auto-download: newly subscribed shows join the next pass.
|
||||
runAutoDownload();
|
||||
return newFeed;
|
||||
})(), "Subscribing");
|
||||
};
|
||||
|
||||
/** Download the N most recent episodes of every in-scope show, per the
|
||||
@@ -332,10 +453,13 @@ function createFeedStore() {
|
||||
};
|
||||
|
||||
/** Apply a freshly fetched episode list to one feed, bumping `lastUpdated`
|
||||
* only when the content actually changed (see sameEpisodes). Returns the
|
||||
* ORIGINAL array reference when nothing changed so callers skip
|
||||
* persistence entirely — a refresh that fetched identical episodes must
|
||||
* not re-sort the "updated" view. */
|
||||
* only when the content actually changed (see sameRefreshWindow). The
|
||||
* fetched window is MERGED into the existing episodes (fetched copy wins
|
||||
* on id collision) so a refresh never shrinks the in-memory list; the
|
||||
* union is capped at MAX_EPISODES_IN_MEMORY. Returns the ORIGINAL array
|
||||
* reference when nothing changed so callers skip persistence entirely —
|
||||
* a refresh that fetched identical episodes must not re-sort the
|
||||
* "updated" view. */
|
||||
const applyRefreshedEpisodes = (
|
||||
prev: Feed[],
|
||||
feedId: string,
|
||||
@@ -344,66 +468,73 @@ function createFeedStore() {
|
||||
let changed = false;
|
||||
const updated = prev.map((f) => {
|
||||
if (f.id !== feedId) return f;
|
||||
if (sameEpisodes(f.episodes, episodes)) return f;
|
||||
const merged = mergeEpisodes(f.episodes, episodes, MAX_EPISODES_IN_MEMORY);
|
||||
if (sameRefreshWindow(f.episodes, episodes)) return f;
|
||||
changed = true;
|
||||
return { ...f, episodes, lastUpdated: new Date() };
|
||||
return { ...f, episodes: merged, lastUpdated: new Date() };
|
||||
});
|
||||
return changed ? updated : prev;
|
||||
};
|
||||
|
||||
/** Refresh a single feed - re-fetch latest 50 episodes */
|
||||
const refreshFeed = async (feedId: string) => {
|
||||
const feed = getFeed(feedId);
|
||||
if (!feed) return;
|
||||
const episodes = await fetchEpisodes(
|
||||
feed.podcast.feedUrl,
|
||||
MAX_EPISODES_REFRESH,
|
||||
feedId,
|
||||
);
|
||||
// Fetch failed (null): keep the currently loaded episodes untouched.
|
||||
if (!episodes) return;
|
||||
setFeeds((prev) => {
|
||||
const updated = applyRefreshedEpisodes(prev, feedId, episodes);
|
||||
if (updated !== prev) saveFeeds(updated);
|
||||
return updated;
|
||||
});
|
||||
const activity = useActivityStore();
|
||||
return activity.track((async () => {
|
||||
const feed = getFeed(feedId);
|
||||
if (!feed) return;
|
||||
const episodes = await fetchEpisodes(
|
||||
feed.podcast.feedUrl,
|
||||
MAX_EPISODES_REFRESH,
|
||||
feedId,
|
||||
);
|
||||
// Fetch failed (null): keep the currently loaded episodes untouched.
|
||||
if (!episodes) return;
|
||||
setFeeds((prev) => {
|
||||
const updated = applyRefreshedEpisodes(prev, feedId, episodes);
|
||||
if (updated !== prev) scheduleSaveFeeds();
|
||||
return updated;
|
||||
});
|
||||
|
||||
// Global auto-download: ensure the N most recent episodes of in-scope
|
||||
// shows are available offline after every refresh (idempotent).
|
||||
runAutoDownload();
|
||||
// Global auto-download: ensure the N most recent episodes of in-scope
|
||||
// shows are available offline after every refresh (idempotent).
|
||||
runAutoDownload();
|
||||
})(), "Refreshing");
|
||||
};
|
||||
|
||||
/** Refresh all feeds — fetch every feed in parallel, then apply ONE
|
||||
* atomic update. Per-feed incremental setFeeds re-sorted the list once
|
||||
* per completion (each refresh bumped lastUpdated and the "updated" sort
|
||||
* re-ran), which showed up as the list order flapping until the batch
|
||||
* finished. */
|
||||
/** Refresh all feeds — bounded concurrency (at most FETCH_CONCURRENCY
|
||||
* in-flight requests), and each feed's refreshed episodes are applied
|
||||
* AS ITS OWN FETCH LANDS (no Promise.all barrier). Per-feed apply is
|
||||
* safe because applyRefreshedEpisodes keeps unchanged feeds' object
|
||||
* identity and lastUpdated (union merge), so each feed's refreshed
|
||||
* episodes render as its own fetch resolves — the order flapping the
|
||||
* old atomic barrier existed to hide can no longer happen. */
|
||||
const refreshAllFeeds = async () => {
|
||||
setIsLoadingFeeds(true);
|
||||
try {
|
||||
const currentFeeds = feeds();
|
||||
const results = await Promise.all(
|
||||
currentFeeds.map(async (feed) => [
|
||||
feed.id,
|
||||
await fetchEpisodes(
|
||||
await mapWithConcurrency(
|
||||
feeds(),
|
||||
FETCH_CONCURRENCY,
|
||||
async (feed) => {
|
||||
const episodes = await fetchEpisodes(
|
||||
feed.podcast.feedUrl,
|
||||
MAX_EPISODES_REFRESH,
|
||||
feed.id,
|
||||
),
|
||||
] as const),
|
||||
);
|
||||
setFeeds((prev) => {
|
||||
let updated = prev;
|
||||
for (const [feedId, episodes] of results) {
|
||||
);
|
||||
// A failed fetch (null) leaves that feed untouched.
|
||||
if (!episodes) continue;
|
||||
updated = applyRefreshedEpisodes(updated, feedId, episodes);
|
||||
}
|
||||
if (updated !== prev) saveFeeds(updated);
|
||||
return updated;
|
||||
});
|
||||
if (!episodes) return;
|
||||
setFeeds((prev) => {
|
||||
const updated = applyRefreshedEpisodes(prev, feed.id, episodes);
|
||||
if (updated !== prev) scheduleSaveFeeds();
|
||||
return updated;
|
||||
});
|
||||
},
|
||||
);
|
||||
// Global auto-download: one idempotent pass after the batch.
|
||||
runAutoDownload();
|
||||
// A refresh batch always ends with a persisted write when
|
||||
// anything changed — never leave the debounce's trailing edge
|
||||
// pending across a process exit.
|
||||
flushPendingSave();
|
||||
} finally {
|
||||
setIsLoadingFeeds(false);
|
||||
}
|
||||
@@ -476,7 +607,10 @@ function createFeedStore() {
|
||||
episodeLoadCount.delete(feedId);
|
||||
setFeeds((prev) => {
|
||||
const updated = prev.filter((f) => f.id !== feedId);
|
||||
saveFeeds(updated);
|
||||
// Unsubscribe intent must not sit in the debounce window if the
|
||||
// process exits — persist the removal immediately.
|
||||
scheduleSaveFeeds();
|
||||
flushPendingSave();
|
||||
return updated;
|
||||
});
|
||||
};
|
||||
@@ -489,7 +623,10 @@ function createFeedStore() {
|
||||
episodeLoadCount.delete(feed.id);
|
||||
setFeeds((prev) => {
|
||||
const updated = prev.filter((f) => f.podcast.feedUrl !== feedUrl);
|
||||
saveFeeds(updated);
|
||||
// Unsubscribe intent must not sit in the debounce window if
|
||||
// the process exits — persist the removal immediately.
|
||||
scheduleSaveFeeds();
|
||||
flushPendingSave();
|
||||
return updated;
|
||||
});
|
||||
}
|
||||
@@ -501,7 +638,7 @@ function createFeedStore() {
|
||||
const updated = prev.map((f) =>
|
||||
f.id === feedId ? { ...f, ...updates, lastUpdated: new Date() } : f,
|
||||
);
|
||||
saveFeeds(updated);
|
||||
scheduleSaveFeeds();
|
||||
return updated;
|
||||
});
|
||||
};
|
||||
@@ -512,7 +649,7 @@ function createFeedStore() {
|
||||
const updated = prev.map((f) =>
|
||||
f.id === feedId ? { ...f, isPinned: !f.isPinned } : f,
|
||||
);
|
||||
saveFeeds(updated);
|
||||
scheduleSaveFeeds();
|
||||
return updated;
|
||||
});
|
||||
};
|
||||
@@ -606,16 +743,28 @@ function createFeedStore() {
|
||||
|
||||
// If no cache, re-fetch and parse the full feed
|
||||
if (!cached) {
|
||||
const response = await fetch(feed.podcast.feedUrl, {
|
||||
headers: {
|
||||
"Accept-Encoding": "identity",
|
||||
Accept: "application/rss+xml, application/xml, text/xml, */*",
|
||||
},
|
||||
});
|
||||
if (!response.ok) return;
|
||||
const xml = await response.text();
|
||||
const parsed = parseRSSFeed(xml, feed.podcast.feedUrl);
|
||||
cached = parsed.episodes;
|
||||
try {
|
||||
const response = await fetch(feed.podcast.feedUrl, {
|
||||
headers: {
|
||||
"Accept-Encoding": "identity",
|
||||
Accept: "application/rss+xml, application/xml, text/xml, */*",
|
||||
},
|
||||
// A hung feed must not stall the load-more path forever —
|
||||
// mirror fetchEpisodes' per-feed timeout.
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!response.ok) return;
|
||||
const xml = await response.text();
|
||||
cached = await parseEpisodesIncremental(xml, feed.podcast.feedUrl);
|
||||
} catch {
|
||||
// Failed/hung refetch: leave the feed's loaded episodes
|
||||
// untouched rather than throwing out of loadMoreEpisodes.
|
||||
return;
|
||||
}
|
||||
// Cold-refetch parse output is unsorted; sort and cap it so the
|
||||
// cache and the pagination window stay newest-first and bounded.
|
||||
cached = sortEpisodesReverseChronological(cached);
|
||||
cached = cached.slice(0, MAX_EPISODES_IN_MEMORY);
|
||||
fullEpisodeCache.set(feedId, cached);
|
||||
// Set current load count to match what's already displayed
|
||||
episodeLoadCount.set(feedId, feed.episodes.length);
|
||||
@@ -636,7 +785,7 @@ function createFeedStore() {
|
||||
const updated = prev.map((f) =>
|
||||
f.id === feedId ? { ...f, episodes } : f,
|
||||
);
|
||||
saveFeeds(updated);
|
||||
scheduleSaveFeeds();
|
||||
return updated;
|
||||
});
|
||||
};
|
||||
@@ -713,6 +862,7 @@ function createFeedStore() {
|
||||
loadMoreEpisodes,
|
||||
loadMoreAllFeeds,
|
||||
hasMoreAcrossAll,
|
||||
flushPendingSave,
|
||||
addSource,
|
||||
removeSource,
|
||||
toggleSource,
|
||||
|
||||
@@ -93,6 +93,13 @@ export function updateConfig(patch: Partial<PodTuiConfig>): void {
|
||||
});
|
||||
}
|
||||
|
||||
/** Resolve once every queued config write has flushed. Tests await this to
|
||||
* observe the serialized result of pending saveFeedsToFile/updateConfig
|
||||
* calls before asserting on config.json. */
|
||||
export function whenConfigIdle(): Promise<void> {
|
||||
return writeChain;
|
||||
}
|
||||
|
||||
/** Guards so migration runs exactly once per process. */
|
||||
let migrationDone = false;
|
||||
let migrationPromise: Promise<void> | null = null;
|
||||
|
||||
26
src/utils/episode-merge.ts
Normal file
26
src/utils/episode-merge.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import type { Episode } from "../types/episode"
|
||||
|
||||
/** Sort key for an episode's pubDate — missing/invalid dates sort as NEWEST
|
||||
* (Infinity) so undated episodes float to the top instead of dropping into
|
||||
* the oldest slot. */
|
||||
const ts = (ep: Episode): number => {
|
||||
const t = ep.pubDate?.getTime()
|
||||
return t === undefined || Number.isNaN(t) ? Infinity : t
|
||||
}
|
||||
|
||||
/**
|
||||
* Union of two episode lists keyed by id — on collision the fetched copy
|
||||
* wins (fresh metadata). Result is sorted newest-first by pubDate and capped
|
||||
* at `cap` entries (oldest dropped). Never mutates either input.
|
||||
*/
|
||||
export function mergeEpisodes(
|
||||
existing: Episode[],
|
||||
fetched: Episode[],
|
||||
cap: number,
|
||||
): Episode[] {
|
||||
const byId = new Map<string, Episode>()
|
||||
for (const ep of existing) byId.set(ep.id, ep)
|
||||
for (const ep of fetched) byId.set(ep.id, ep)
|
||||
const sorted = [...byId.values()].sort((a, b) => ts(b) - ts(a))
|
||||
return sorted.slice(0, cap)
|
||||
}
|
||||
@@ -4,9 +4,57 @@
|
||||
*/
|
||||
|
||||
import { loadConfig, updateConfig } from "./config";
|
||||
import { getConfigFilePath } from "./config-dir";
|
||||
import { DownloadStatus } from "../types/episode";
|
||||
import type { Episode } from "../types/episode";
|
||||
import type { Feed } from "../types/feed";
|
||||
import type { PodcastSource } from "../types/source";
|
||||
|
||||
/** Retention window for persisted episodes: older episodes are dropped when
|
||||
* feeds are written to config.json unless they are completed downloads. */
|
||||
export const PERSISTED_WINDOW_DAYS = 30;
|
||||
|
||||
/** True when an episode may be persisted: it is a completed download, or its
|
||||
* pubDate is missing/invalid (fail-safe: never drop an undatable episode),
|
||||
* or it falls inside the retention window. */
|
||||
export function episodeIsPersistable(
|
||||
ep: Episode,
|
||||
downloadedIds: Set<string>,
|
||||
now: Date,
|
||||
): boolean {
|
||||
if (downloadedIds.has(ep.id)) return true;
|
||||
const t = ep.pubDate?.getTime();
|
||||
if (!t || Number.isNaN(t)) return true;
|
||||
return t >= now.getTime() - PERSISTED_WINDOW_DAYS * 24 * 3600 * 1000;
|
||||
}
|
||||
|
||||
/** Episode ids of completed downloads, read from downloads.json. In-flight
|
||||
* downloads are NOT exempted from the retention window — a just-completed
|
||||
* download is re-included by the next save because the in-memory
|
||||
* feed.episodes still holds it. Missing/unreadable/invalid file → empty set. */
|
||||
async function readDownloadedEpisodeIds(): Promise<Set<string>> {
|
||||
try {
|
||||
const file = Bun.file(getConfigFilePath("downloads.json"));
|
||||
if (!(await file.exists())) return new Set();
|
||||
const raw = await file.json();
|
||||
if (!Array.isArray(raw)) return new Set();
|
||||
const ids = new Set<string>();
|
||||
for (const rec of raw) {
|
||||
if (
|
||||
rec &&
|
||||
typeof rec === "object" &&
|
||||
rec.status === DownloadStatus.COMPLETED &&
|
||||
typeof rec.episodeId === "string"
|
||||
) {
|
||||
ids.add(rec.episodeId);
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
/** Deserialize date strings back to Date objects in feed data */
|
||||
function reviveDates(feed: Feed): Feed {
|
||||
return {
|
||||
@@ -23,20 +71,54 @@ function reviveDates(feed: Feed): Feed {
|
||||
};
|
||||
}
|
||||
|
||||
/** Load feeds from config.json */
|
||||
/** Load feeds from config.json, pruning episodes outside the retention
|
||||
* window (completed downloads always kept). When anything was pruned, the
|
||||
* pruned list is rewritten to config.json (startup cleanup for legacy
|
||||
* configs). The read path is awaited so the returned value is deterministic. */
|
||||
export async function loadFeedsFromFile(): Promise<Feed[]> {
|
||||
try {
|
||||
const cfg = await loadConfig();
|
||||
if (!Array.isArray(cfg.feeds)) return [];
|
||||
return cfg.feeds.map(reviveDates);
|
||||
const feeds = cfg.feeds.map(reviveDates);
|
||||
const downloadedIds = await readDownloadedEpisodeIds();
|
||||
const now = new Date();
|
||||
let prunedAny = false;
|
||||
const pruned = feeds.map((f) => {
|
||||
const kept = f.episodes.filter((ep) =>
|
||||
episodeIsPersistable(ep, downloadedIds, now),
|
||||
);
|
||||
if (kept.length !== f.episodes.length) prunedAny = true;
|
||||
return { ...f, episodes: kept };
|
||||
});
|
||||
if (prunedAny) {
|
||||
// Fire-and-forget cleanup rewrite of the legacy config.
|
||||
saveFeedsToFile(pruned);
|
||||
}
|
||||
return pruned;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Save feeds to config.json */
|
||||
/** Save feeds to config.json, pruning episodes outside the retention window
|
||||
* (completed downloads always kept). Fire-and-forget: the prune reads
|
||||
* downloads.json asynchronously, then enqueues the write. On any error the
|
||||
* UNPRUNED feeds are saved instead, so data is never lost. */
|
||||
export function saveFeedsToFile(feeds: Feed[]): void {
|
||||
updateConfig({ feeds });
|
||||
(async () => {
|
||||
try {
|
||||
const downloadedIds = await readDownloadedEpisodeIds();
|
||||
const pruned = feeds.map((f) => ({
|
||||
...f,
|
||||
episodes: f.episodes.filter((ep) =>
|
||||
episodeIsPersistable(ep, downloadedIds, new Date()),
|
||||
),
|
||||
}));
|
||||
updateConfig({ feeds: pruned });
|
||||
} catch {
|
||||
updateConfig({ feeds }); /* never lose data on an error path */
|
||||
}
|
||||
})().catch(() => {});
|
||||
}
|
||||
|
||||
/** Load sources from config.json */
|
||||
|
||||
Reference in New Issue
Block a user