refactor(feed): port refresh batch to Effect for concurrency and timeout
Replace the hand-rolled worker pool (mapWithConcurrency) with an Effect program (src/effects/feed-refresh.ts): Effect.forEach bounds in-flight fetches, Effect.timeout bounds each feed via the Clock service, and failures fold to null so a bad feed never fails the batch. Per-feed apply-as-it-lands is preserved — the apply callback runs inside each feed's own fiber, so there is no Promise.all barrier. Store boundary unchanged: refreshAllFeeds runs the program through Effect.runPromise, keeping runAutoDownload + flushPendingSave after the batch and isLoadingFeeds around it. Adds TestClock-driven tests (tests/feed-refresh-effect.test.ts) that pin concurrency, per-feed apply, timeout, and failure isolation without real 20s waits. Pins effect@^3 (V4 is in beta).
This commit is contained in:
85
src/effects/feed-refresh.ts
Normal file
85
src/effects/feed-refresh.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Feed-refresh batch as an Effect program.
|
||||
*
|
||||
* Replaces the hand-rolled worker pool (mapWithConcurrency) + per-feed
|
||||
* fetch/apply plumbing in stores/feed.ts with Effect's structured
|
||||
* concurrency:
|
||||
* - `Effect.forEach(..., { concurrency })` bounds in-flight fetches to
|
||||
* `concurrency` (starts exactly that many fibers; each completion pulls
|
||||
* the next feed — identical semantics to the old shared-counter pool).
|
||||
* - `Effect.timeout` bounds each feed's fetch to `timeoutMs`. It runs
|
||||
* through the `Clock` service, so under `TestContext` the TestClock
|
||||
* drives it deterministically (no real 20s wait in tests).
|
||||
* - Failures are folded to a null result: a failed or timed-out feed is
|
||||
* left untouched instead of failing the batch.
|
||||
* - The apply callback runs inside each feed's own fiber, so a feed's
|
||||
* refreshed episodes land AS ITS OWN FETCH COMPLETES — the
|
||||
* per-feed-apply-as-it-lands contract, no Promise.all barrier.
|
||||
*
|
||||
* The store boundary (stores/feed.ts) supplies the real fetch and apply
|
||||
* closures and runs the program with Effect.runPromise.
|
||||
*/
|
||||
|
||||
import { Duration, Effect } from "effect"
|
||||
import type { Episode } from "../types/episode"
|
||||
import type { Feed } from "../types/feed"
|
||||
|
||||
/** Result of fetching one feed's RSS. `episodes: null` means the fetch
|
||||
* failed or timed out — callers must leave that feed untouched. */
|
||||
export interface RefreshFetchResult {
|
||||
episodes: Episode[] | null
|
||||
coverUrl: string | undefined
|
||||
}
|
||||
|
||||
/** Result guaranteed to have parsed episodes (the apply path only). */
|
||||
export interface RefreshSuccess {
|
||||
episodes: Episode[]
|
||||
coverUrl: string | undefined
|
||||
}
|
||||
|
||||
export interface RefreshBatchOptions {
|
||||
/** Max simultaneous in-flight fetches. */
|
||||
concurrency: number
|
||||
/** Per-feed fetch timeout in milliseconds. */
|
||||
timeoutMs: number
|
||||
}
|
||||
|
||||
/** Fold any failure (network error, timeout, rejection) to a null result so
|
||||
* one bad feed can never fail the batch. */
|
||||
const failedResult: RefreshFetchResult = { episodes: null, coverUrl: undefined }
|
||||
|
||||
/** Fetch one feed with a timeout, applying its result as its own fetch
|
||||
* lands. A failed or timed-out fetch yields null — the feed is untouched. */
|
||||
const refreshOne = (
|
||||
feed: Feed,
|
||||
fetchOne: (feed: Feed) => Promise<RefreshFetchResult>,
|
||||
applyOne: (feed: Feed, result: RefreshSuccess) => void,
|
||||
timeoutMs: number,
|
||||
): Effect.Effect<void> =>
|
||||
Effect.tryPromise(() => fetchOne(feed)).pipe(
|
||||
Effect.timeout(Duration.millis(timeoutMs)),
|
||||
Effect.catchAll(() => Effect.succeed(failedResult)),
|
||||
Effect.flatMap((result) => {
|
||||
if (result.episodes === null) return Effect.void
|
||||
// Capture the narrowed array before the closure — TS drops the
|
||||
// `episodes !== null` narrowing inside Effect.sync's callback.
|
||||
const episodes = result.episodes
|
||||
return Effect.sync(() => applyOne(feed, { episodes, coverUrl: result.coverUrl }))
|
||||
}),
|
||||
)
|
||||
|
||||
/** Refresh every feed with bounded concurrency. Each feed's refreshed
|
||||
* episodes are applied as its own fetch lands (no barrier); a failed or
|
||||
* timed-out feed is left untouched. The program never fails — failures
|
||||
* are folded to per-feed no-ops. */
|
||||
export const refreshFeedsBatch = (
|
||||
feeds: readonly Feed[],
|
||||
fetchOne: (feed: Feed) => Promise<RefreshFetchResult>,
|
||||
applyOne: (feed: Feed, result: RefreshSuccess) => void,
|
||||
options: RefreshBatchOptions,
|
||||
): Effect.Effect<void> =>
|
||||
Effect.forEach(
|
||||
feeds,
|
||||
(feed) => refreshOne(feed, fetchOne, applyOne, options.timeoutMs),
|
||||
{ concurrency: options.concurrency, discard: true },
|
||||
)
|
||||
@@ -4,6 +4,8 @@
|
||||
*/
|
||||
|
||||
import { createSignal } from "solid-js";
|
||||
import { Effect } from "effect";
|
||||
import { refreshFeedsBatch } from "../effects/feed-refresh";
|
||||
import { FeedVisibility } from "../types/feed";
|
||||
import type { Feed, FeedFilter, FeedSortField } from "../types/feed";
|
||||
import type { Podcast } from "../types/podcast";
|
||||
@@ -249,31 +251,6 @@ export function sameRefreshWindow(
|
||||
return fetched.every((e) => signatures.has(episodeSignature(e)));
|
||||
}
|
||||
|
||||
/** Run `fn` over every item with at most `limit` executions in flight — a
|
||||
* classic worker pool. Workers pull indexes from a shared counter, so the
|
||||
* first `limit` calls start immediately and each completion frees its slot
|
||||
* for the next item; results are assembled in INPUT order regardless of
|
||||
* completion order. A hung `fn` holds at most one slot. */
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results = new Array<R>(items.length);
|
||||
let nextIndex = 0;
|
||||
const workers = Array.from(
|
||||
{ length: Math.min(limit, items.length) },
|
||||
async () => {
|
||||
let i: number;
|
||||
while ((i = nextIndex++) < items.length) {
|
||||
results[i] = await fn(items[i]);
|
||||
}
|
||||
},
|
||||
);
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
function createFeedStore() {
|
||||
const [feeds, setFeeds] = createSignal<Feed[]>([]);
|
||||
const [sources, setSources] = createSignal<PodcastSource[]>([
|
||||
@@ -644,40 +621,40 @@ function createFeedStore() {
|
||||
})(), "Refreshing");
|
||||
};
|
||||
|
||||
/** Refresh all feeds — bounded concurrency (at most FETCH_CONCURRENCY
|
||||
* in-flight requests), and each feed's refreshed episodes are applied
|
||||
* AS ITS OWN FETCH LANDS (no Promise.all barrier). Per-feed apply is
|
||||
* safe because applyRefreshedEpisodes keeps unchanged feeds' object
|
||||
* identity and lastUpdated (union merge), so each feed's refreshed
|
||||
* episodes render as its own fetch resolves — the order flapping the
|
||||
* old atomic barrier existed to hide can no longer happen. */
|
||||
/** Refresh all feeds via the Effect batch program (effects/feed-refresh):
|
||||
* bounded concurrency (at most FETCH_CONCURRENCY in-flight requests)
|
||||
* and each feed's refreshed episodes applied AS ITS OWN FETCH LANDS
|
||||
* (no barrier — the apply runs inside the feed's own fiber). Per-feed
|
||||
* apply is safe because applyRefreshedEpisodes keeps unchanged feeds'
|
||||
* object identity and lastUpdated (union merge), so each feed's
|
||||
* refreshed episodes render as its own fetch resolves — the order
|
||||
* flapping the old atomic barrier existed to hide can no longer
|
||||
* happen. A failed or timed-out fetch (null episodes) leaves that
|
||||
* feed untouched. */
|
||||
const refreshAllFeeds = async () => {
|
||||
setIsLoadingFeeds(true);
|
||||
try {
|
||||
await mapWithConcurrency(
|
||||
feeds(),
|
||||
FETCH_CONCURRENCY,
|
||||
async (feed) => {
|
||||
const { episodes, coverUrl } = await fetchEpisodes(
|
||||
feed.podcast.feedUrl,
|
||||
MAX_EPISODES_REFRESH,
|
||||
feed.id,
|
||||
);
|
||||
// A failed fetch (null) leaves that feed untouched.
|
||||
if (!episodes) return;
|
||||
setFeeds((prev) => {
|
||||
let updated = applyRefreshedEpisodes(prev, feed.id, episodes);
|
||||
if (coverUrl) {
|
||||
updated = updated.map((f) =>
|
||||
f.id === feed.id && !f.podcast.coverUrl && coverUrl
|
||||
? { ...f, podcast: { ...f.podcast, coverUrl } }
|
||||
: f,
|
||||
);
|
||||
}
|
||||
if (updated !== prev) scheduleSaveFeeds();
|
||||
return updated;
|
||||
});
|
||||
},
|
||||
await Effect.runPromise(
|
||||
refreshFeedsBatch(
|
||||
feeds(),
|
||||
(feed) =>
|
||||
fetchEpisodes(feed.podcast.feedUrl, MAX_EPISODES_REFRESH, feed.id),
|
||||
(feed, { episodes, coverUrl }) => {
|
||||
setFeeds((prev) => {
|
||||
let updated = applyRefreshedEpisodes(prev, feed.id, episodes);
|
||||
if (coverUrl) {
|
||||
updated = updated.map((f) =>
|
||||
f.id === feed.id && !f.podcast.coverUrl && coverUrl
|
||||
? { ...f, podcast: { ...f.podcast, coverUrl } }
|
||||
: f,
|
||||
);
|
||||
}
|
||||
if (updated !== prev) scheduleSaveFeeds();
|
||||
return updated;
|
||||
});
|
||||
},
|
||||
{ concurrency: FETCH_CONCURRENCY, timeoutMs: FETCH_TIMEOUT_MS },
|
||||
),
|
||||
);
|
||||
// Global auto-download: one idempotent pass after the batch.
|
||||
runAutoDownload();
|
||||
|
||||
Reference in New Issue
Block a user