fix(search): surface iTunes stubs and resolve feeds for delisted shows

Shows that left Apple Podcasts (e.g. Daily Wire's in 2021) come back from
the iTunes Search API as metadata-only stub records with feedUrl null.
mapItunesResult dropped them, so The Ben Shapiro Show — the #1 hit for
'ben shapiro' — never appeared in search while sibling shows did.

- Keep feedUrl-less results (feedUrl "" + directoryUrl pointing at the
  Apple page) so delisted shows stay findable.
- Resolve the real feed from the Apple page at subscribe time
  (itunes-feed-resolver: anchor on the collection's adamId, forward-scan
  for the embedded feedUrl; Apple serves page variants where the
  showOffer block sits thousands of chars after the adamId).
- addFeed refuses feedless stubs whose feed can't be resolved instead of
  adding a broken feed; SearchPage surfaces the failure via toast.
- Tests: stub mapping, extractor variants, and an end-to-end subscribe
  over a local HTTP server.
This commit is contained in:
2026-08-10 22:38:03 -04:00
parent e73e608b9f
commit 0b0637b9dc
8 changed files with 364 additions and 11 deletions

View File

@@ -26,6 +26,7 @@ import {
} from "solid-js";
import { useSearchStore } from "@/stores/search";
import { useFeedStore } from "@/stores/feed";
import { useToast } from "@/ui/toast";
import { format } from "date-fns";
import { useTheme } from "@/context/ThemeContext";
import {
@@ -49,6 +50,7 @@ export const SearchPaneCount = 1;
function SearchPage() {
const searchStore = useSearchStore();
const feedStore = useFeedStore();
const toast = useToast();
const [inputValue, setInputValue] = createSignal("");
const { theme } = useTheme();
const muted = () => theme.muted || theme.text;
@@ -136,10 +138,23 @@ function SearchPage() {
runSearch(query);
};
const handleSubscribe = (result: SearchResult) => {
// Actually add the feed to the feed store, then mark the result subscribed
feedStore.addFeed(result.podcast, result.sourceId).catch(() => {});
searchStore.markSubscribed(result.podcast.id);
const handleSubscribe = async (result: SearchResult) => {
// Actually add the feed to the feed store, then mark the result
// subscribed. addFeed returns null when a feedless directory stub
// (delisted show) can't be resolved — tell the user why.
const feed = await feedStore
.addFeed(result.podcast, result.sourceId)
.catch(() => null);
if (!feed && !result.podcast.feedUrl) {
toast.show({
title: "Can't subscribe",
message:
"No RSS feed is listed for this show and the feed couldn't be resolved. Try adding it by feed URL.",
variant: "error",
});
return;
}
if (feed) searchStore.markSubscribed(result.podcast.id);
};
// ── nav.action handler ──────────────────────────────────────────────────────
@@ -450,7 +465,11 @@ function SearchPage() {
</For>
</box>
</Show>
<text fg={muted()}>Feed: {result().podcast.feedUrl}</text>
<text fg={muted()}>
Feed:{" "}
{result().podcast.feedUrl ||
"not listed by source — resolves on subscribe"}
</text>
<text fg={muted()}>
Updated: {formatDate(result().podcast.lastUpdated)}
</text>

View File

@@ -11,6 +11,7 @@ import type { Episode } from "../types/episode";
import type { PodcastSource } from "../types/source";
import { DEFAULT_SOURCES } from "../types/source";
import { parseRSSFeed } from "../api/rss-parser";
import { resolveItunesFeedUrl } from "../utils/itunes-feed-resolver";
import {
loadFeedsFromFile,
saveFeedsToFile,
@@ -196,6 +197,17 @@ function createFeedStore() {
sourceId: string,
visibility: FeedVisibility = FeedVisibility.PUBLIC,
): Promise<Feed | null> => {
// A directory stub (e.g. a show delisted from Apple Podcasts) has no
// feed URL; resolve the real feed from its directory page before
// subscribing. Refuse when it can't be resolved rather than adding a
// broken feed.
if (!podcast.feedUrl) {
if (!podcast.directoryUrl) return null;
const resolved = await resolveItunesFeedUrl(podcast.directoryUrl);
if (!resolved) return null;
podcast = { ...podcast, feedUrl: resolved, directoryUrl: undefined };
}
// Guard: don't add a feed we already have (matched by feedUrl)
if (hasFeedByUrl(podcast.feedUrl)) {
return feeds().find((f) => f.podcast.feedUrl === podcast.feedUrl) ?? null;

View File

@@ -12,8 +12,12 @@ export interface Podcast {
description: string
/** Cover image URL */
coverUrl?: string
/** RSS feed URL */
/** RSS feed URL. Empty when the directory lists the show without a feed
* (e.g. shows delisted from Apple Podcasts); see directoryUrl. */
feedUrl: string
/** Directory listing page (e.g. Apple Podcasts) for shows whose feed URL
* the directory omits — used to resolve the real feed at subscribe time. */
directoryUrl?: string
/** Author/creator name */
author?: string
/** Podcast categories */

View File

@@ -0,0 +1,61 @@
/**
* iTunes feed resolution for shows delisted from Apple Podcasts.
*
* The iTunes Search API returns `feedUrl: null` for shows that left Apple
* Podcasts (e.g. The Daily Wire's shows in 2021) — the directory keeps a
* metadata-only stub. The show's public Apple Podcasts page still embeds the
* real feed URL in its JSON state (`showOffer.feedUrl`), so subscribing can
* resolve it from there.
*/
/** `"feedUrl":"https://..."` as embedded in the Apple page's JSON state. */
const FEED_URL_RE = /"feedUrl"\s*:\s*"(https?:\/\/[^"]+)"/
/**
* Extract the show's feed URL from an Apple Podcasts page's HTML.
*
* The page embeds `showOffer` blocks for the show AND for related shows, each
* with its own feedUrl, and Apple serves multiple JSON variants — the main
* show's showOffer may sit adjacent to its adamId or thousands of chars later.
* Anchor on the collection id from `directoryUrl` (`"adamId":"<id>"`) and take
* the FIRST feedUrl after it (the main show's content precedes related shows'
* in the document). Falls back to the first feedUrl in the document only when
* the id isn't present in the URL. Returns null when no trustworthy match
* exists (page restructured, no feed) — callers must not guess.
*/
export const extractFeedUrlFromPage = (
html: string,
directoryUrl: string,
): string | null => {
const idMatch = /[?/]id(\d+)/.exec(directoryUrl)
if (!idMatch) {
const fallback = FEED_URL_RE.exec(html)
return fallback ? fallback[1] : null
}
const adamIdx = html.search(new RegExp(`"adamId"\\s*:\\s*"${idMatch[1]}"`))
if (adamIdx < 0) return null
const fromAdam = new RegExp(FEED_URL_RE.source, "g")
fromAdam.lastIndex = adamIdx
const match = fromAdam.exec(html)
return match ? match[1] : null
}
/**
* Resolve a delisted show's RSS feed from its Apple Podcasts page.
* Returns null on network failure or when the page has no resolvable feed.
*/
export const resolveItunesFeedUrl = async (
directoryUrl: string,
): Promise<string | null> => {
try {
const response = await fetch(directoryUrl, {
headers: { "User-Agent": "PodTUI/1.0" },
})
if (!response.ok) return null
return extractFeedUrlFromPage(await response.text(), directoryUrl)
} catch {
return null
}
}

View File

@@ -14,11 +14,13 @@ type ItunesResult = {
collectionId?: number
collectionName?: string
artistName?: string
feedUrl?: string
/** Null for shows delisted from Apple Podcasts (directory stub records). */
feedUrl?: string | null
artworkUrl100?: string
artworkUrl600?: string
primaryGenreName?: string
releaseDate?: string
collectionViewUrl?: string
}
type ItunesResponse = {
@@ -41,8 +43,8 @@ const buildItunesUrl = (query: string, source: PodcastSource) => {
return url.toString()
}
const mapItunesResult = (result: ItunesResult, source: PodcastSource): Podcast | null => {
if (!result.collectionName || !result.feedUrl) return null
export const mapItunesResult = (result: ItunesResult, source: PodcastSource): Podcast | null => {
if (!result.collectionName) return null
const id = result.collectionId
? `itunes-${result.collectionId}`
@@ -52,11 +54,18 @@ const mapItunesResult = (result: ItunesResult, source: PodcastSource): Podcast |
if (result.artistName) descriptionParts.push(`by ${result.artistName}`)
if (result.primaryGenreName) descriptionParts.push(result.primaryGenreName)
// Shows delisted from Apple Podcasts (e.g. The Daily Wire's shows) come back
// as metadata-only stub records with feedUrl null. Keep the stub so the show
// stays findable; the real feed is resolved from the directory page at
// subscribe time (see itunes-feed-resolver).
const feedUrl = result.feedUrl ?? ""
return {
id,
title: result.collectionName,
description: descriptionParts.join(" • "),
feedUrl: result.feedUrl,
feedUrl,
directoryUrl: feedUrl ? undefined : result.collectionViewUrl,
author: result.artistName,
categories: result.primaryGenreName ? [result.primaryGenreName] : undefined,
coverUrl: result.artworkUrl600 || result.artworkUrl100,

View File

@@ -0,0 +1,117 @@
/**
* End-to-end subscribe test for feedless directory stubs.
*
* A delisted show (feedUrl "" + directoryUrl) must resolve its real feed from
* the directory page inside addFeed — so subscribing just works. When the
* page can't be resolved, addFeed must refuse (return null) instead of adding
* a broken feed. Served over a real local HTTP server, mirroring how the
* app's other store tests exercise the network path.
*/
import { test, expect, beforeAll, afterAll } from "bun:test";
import { mkdtempSync, rmSync } from "fs";
import { tmpdir } from "os";
import { join } from "path";
// Point the config dir at a throwaway directory BEFORE importing the stores
// (their module-level init reads it).
const configHome = mkdtempSync(join(tmpdir(), "podtui-feedless-"));
process.env.XDG_CONFIG_HOME = configHome;
import { useFeedStore } from "../src/stores/feed";
import type { Podcast } from "../src/types/podcast";
const FEED_ID = "12345";
const EPISODE_TITLES = ["Ep 2", "Ep 1"];
function pageHtml(feedUrl: string): string {
return `<html><body><script>
{"pageData":{"showOffer":{"title":"Delisted Show","adamId":"${FEED_ID}","feedUrl":"${feedUrl}","showType":"episodic"}}}
</script></body></html>`;
}
function feedXml(origin: string): string {
const items = EPISODE_TITLES.map(
(title, i) => `<item>
<title>${title}</title>
<pubDate>2026-08-0${2 - i}T00:00:00Z</pubDate>
<enclosure url="${origin}/audio-${i}.mp3" length="12345" type="audio/mpeg"/>
</item>`,
).join("\n");
return `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"><channel>
<title>Delisted Show</title>
<description>Feedless stub test</description>
${items}
</channel></rss>`;
}
let server: ReturnType<typeof Bun.serve> | null = null;
let feedUrl = "";
let pageUrl = "";
beforeAll(() => {
server = Bun.serve({
port: 0,
fetch(req) {
const url = new URL(req.url);
if (url.pathname.endsWith(".rss")) {
return new Response(feedXml(url.origin), {
headers: { "Content-Type": "application/rss+xml" },
});
}
if (url.pathname.startsWith("/show")) {
return new Response(pageHtml(feedUrl), {
headers: { "Content-Type": "text/html" },
});
}
return new Response("not found", { status: 404 });
},
});
feedUrl = `http://127.0.0.1:${server!.port}/feed.rss`;
// The resolver anchors on `/id<digits>` in the directory URL.
pageUrl = `http://127.0.0.1:${server!.port}/show/id${FEED_ID}`;
});
afterAll(() => {
server?.stop(true);
rmSync(configHome, { recursive: true, force: true });
});
const makeStub = (feedUrl: string, directoryUrl?: string): Podcast => ({
id: "itunes-12345",
title: "Delisted Show",
description: "Show that left the directory",
author: "Some Network",
feedUrl,
directoryUrl,
lastUpdated: new Date(),
isSubscribed: false,
});
test("addFeed resolves a feedless stub's feed from its directory page", async () => {
const store = useFeedStore();
const feed = await store.addFeed(makeStub("", pageUrl), "itunes");
expect(feed).not.toBeNull();
expect(feed!.podcast.feedUrl).toBe(feedUrl);
// Resolution metadata is dropped from the persisted feed record.
expect(feed!.podcast.directoryUrl).toBeUndefined();
expect(feed!.episodes.map((e) => e.title)).toEqual(EPISODE_TITLES);
// Remove the feed: bun test shares the store singleton across files, and a
// leftover feed (whose server dies in afterAll) would reorder other files'
// refresh assertions.
store.removeFeed(feed!.id);
});
test("addFeed refuses a stub whose directory page cannot be resolved", async () => {
const store = useFeedStore();
const unreachable = makeStub("", "http://127.0.0.1:1/nope/id999");
const feed = await store.addFeed(unreachable, "itunes");
expect(feed).toBeNull();
});
test("addFeed refuses a stub with no directory page at all", async () => {
const store = useFeedStore();
const bare = makeStub("", undefined);
const feed = await store.addFeed(bare, "itunes");
expect(feed).toBeNull();
});

View File

@@ -0,0 +1,70 @@
/**
* Feed-resolution extraction tests.
*
* The iTunes Search API returns feedUrl null for shows delisted from Apple
* Podcasts. Their public Apple page still embeds the real feed URL in JSON
* state — alongside feedUrls of RELATED shows — so extraction must anchor on
* the show's adamId rather than grabbing the first feedUrl in the document.
*/
import { test, expect } from "bun:test";
import { extractFeedUrlFromPage } from "../src/utils/itunes-feed-resolver";
const MAIN_FEED = "https://rss.pdrl.fm/b32227/feeds.megaphone.fm/BVDWV5370667266";
const OTHER_FEED = "https://feeds.megaphone.fm/BVDWV7762869899";
/** Synthetic Apple page: related shows first, main showOffer after. */
const pageWithNoise = `{
"shows":[{"showOffer":{"title":"The Matt Walsh Show","adamId":"2950206264","feedUrl":"${OTHER_FEED}","showType":"episodic"}}],
"pageData":{"showOffer":{"title":"The Ben Shapiro Show","adamId":"1047335260","feedUrl":"${MAIN_FEED}","showType":"episodic"}}
}`;
test("extracts the show's feed anchored on its adamId, ignoring related shows", () => {
const feed = extractFeedUrlFromPage(
pageWithNoise,
"https://podcasts.apple.com/us/podcast/the-ben-shapiro-show/id1047335260",
);
expect(feed).toBe(MAIN_FEED);
});
test("handles the ?uo=4 suffix Apple appends to directory URLs", () => {
const feed = extractFeedUrlFromPage(
pageWithNoise,
"https://podcasts.apple.com/us/podcast/the-ben-shapiro-show/id1047335260?uo=4",
);
expect(feed).toBe(MAIN_FEED);
});
test("returns null when the page has no feedUrl for the requested id", () => {
const feed = extractFeedUrlFromPage(
pageWithNoise,
"https://podcasts.apple.com/us/podcast/some-other-show/id9999999999",
);
expect(feed).toBeNull();
});
test("finds the feed when the showOffer sits far after the adamId reference", () => {
// Apple serves page variants where thousands of chars separate the first
// adamId reference from the showOffer block carrying the feedUrl.
const variant = `{"adamId":"1047335260","$kind":"ShowPageIntent"}${"x".repeat(6000)}{"showOffer":{"title":"The Ben Shapiro Show","adamId":"1047335260","feedUrl":"${MAIN_FEED}"}}`;
const feed = extractFeedUrlFromPage(
variant,
"https://podcasts.apple.com/us/podcast/the-ben-shapiro-show/id1047335260",
);
expect(feed).toBe(MAIN_FEED);
});
test("falls back to the first feedUrl when the URL carries no id", () => {
const feed = extractFeedUrlFromPage(
pageWithNoise,
"https://podcasts.apple.com/us/podcast/the-ben-shapiro-show",
);
expect(feed).toBe(OTHER_FEED);
});
test("returns null when the page contains no feedUrl at all", () => {
const feed = extractFeedUrlFromPage(
"<html><body>not found</body></html>",
"https://podcasts.apple.com/us/podcast/the-ben-shapiro-show/id1047335260",
);
expect(feed).toBeNull();
});

View File

@@ -9,7 +9,7 @@
* search. This test pins the empty-result contract.
*/
import { test, expect } from "bun:test";
import { searchSourceByType } from "../src/utils/source-searcher";
import { searchSourceByType, mapItunesResult } from "../src/utils/source-searcher";
import { SourceType } from "../src/types/source";
import type { PodcastSource } from "../src/types/source";
@@ -29,6 +29,16 @@ const customSource: PodcastSource = {
enabled: true,
};
const itunesSource: PodcastSource = {
id: "itunes",
name: "Apple Podcasts",
type: SourceType.API,
baseUrl: "https://itunes.apple.com/search",
enabled: true,
country: "US",
language: "en_us",
};
test("RSS sources return no directory search results", async () => {
const results = await searchSourceByType("blocked and reported", rssSource);
expect(results).toEqual([]);
@@ -38,3 +48,54 @@ test("custom sources return no directory search results", async () => {
const results = await searchSourceByType("anything", customSource);
expect(results).toEqual([]);
});
// ── iTunes stub records (delisted shows) ────────────────────────────────────
// Shows that left Apple Podcasts (e.g. The Daily Wire's in 2021) remain in
// the directory as metadata-only records with feedUrl null. They must stay
// findable — earlier they were dropped entirely, so "ben shapiro" surfaced
// nothing while the show is the #1 iTunes hit.
test("iTunes results without a feedUrl (delisted shows) are kept, not dropped", () => {
const stub = mapItunesResult(
{
collectionId: 1047335260,
collectionName: "The Ben Shapiro Show",
artistName: "The Daily Wire",
feedUrl: null,
collectionViewUrl:
"https://podcasts.apple.com/us/podcast/the-ben-shapiro-show/id1047335260",
},
itunesSource,
);
expect(stub).not.toBeNull();
expect(stub!.title).toBe("The Ben Shapiro Show");
// Empty feed marks "unavailable from this directory"; the Apple page URL
// is carried for feed resolution at subscribe time.
expect(stub!.feedUrl).toBe("");
expect(stub!.directoryUrl).toBe(
"https://podcasts.apple.com/us/podcast/the-ben-shapiro-show/id1047335260",
);
});
test("iTunes results with a feedUrl keep it and carry no directory fallback", () => {
const normal = mapItunesResult(
{
collectionId: 1487234816,
collectionName: "Morning Wire",
artistName: "The Daily Wire",
feedUrl: "https://feeds.megaphone.fm/BVDWV8747925072",
},
itunesSource,
);
expect(normal).not.toBeNull();
expect(normal!.feedUrl).toBe("https://feeds.megaphone.fm/BVDWV8747925072");
expect(normal!.directoryUrl).toBeUndefined();
});
test("iTunes results without a collection name stay dropped", () => {
const dropped = mapItunesResult(
{ collectionId: 1, feedUrl: "https://example.com/feed.xml" },
itunesSource,
);
expect(dropped).toBeNull();
});