fix(search): surface iTunes stubs and resolve feeds for delisted shows
Shows that left Apple Podcasts (e.g. Daily Wire's in 2021) come back from the iTunes Search API as metadata-only stub records with feedUrl null. mapItunesResult dropped them, so The Ben Shapiro Show — the #1 hit for 'ben shapiro' — never appeared in search while sibling shows did. - Keep feedUrl-less results (feedUrl "" + directoryUrl pointing at the Apple page) so delisted shows stay findable. - Resolve the real feed from the Apple page at subscribe time (itunes-feed-resolver: anchor on the collection's adamId, forward-scan for the embedded feedUrl; Apple serves page variants where the showOffer block sits thousands of chars after the adamId). - addFeed refuses feedless stubs whose feed can't be resolved instead of adding a broken feed; SearchPage surfaces the failure via toast. - Tests: stub mapping, extractor variants, and an end-to-end subscribe over a local HTTP server.
This commit is contained in:
61
src/utils/itunes-feed-resolver.ts
Normal file
61
src/utils/itunes-feed-resolver.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* iTunes feed resolution for shows delisted from Apple Podcasts.
|
||||
*
|
||||
* The iTunes Search API returns `feedUrl: null` for shows that left Apple
|
||||
* Podcasts (e.g. The Daily Wire's shows in 2021) — the directory keeps a
|
||||
* metadata-only stub. The show's public Apple Podcasts page still embeds the
|
||||
* real feed URL in its JSON state (`showOffer.feedUrl`), so subscribing can
|
||||
* resolve it from there.
|
||||
*/
|
||||
|
||||
/** `"feedUrl":"https://..."` as embedded in the Apple page's JSON state. */
|
||||
const FEED_URL_RE = /"feedUrl"\s*:\s*"(https?:\/\/[^"]+)"/
|
||||
|
||||
/**
|
||||
* Extract the show's feed URL from an Apple Podcasts page's HTML.
|
||||
*
|
||||
* The page embeds `showOffer` blocks for the show AND for related shows, each
|
||||
* with its own feedUrl, and Apple serves multiple JSON variants — the main
|
||||
* show's showOffer may sit adjacent to its adamId or thousands of chars later.
|
||||
* Anchor on the collection id from `directoryUrl` (`"adamId":"<id>"`) and take
|
||||
* the FIRST feedUrl after it (the main show's content precedes related shows'
|
||||
* in the document). Falls back to the first feedUrl in the document only when
|
||||
* the id isn't present in the URL. Returns null when no trustworthy match
|
||||
* exists (page restructured, no feed) — callers must not guess.
|
||||
*/
|
||||
export const extractFeedUrlFromPage = (
|
||||
html: string,
|
||||
directoryUrl: string,
|
||||
): string | null => {
|
||||
const idMatch = /[?/]id(\d+)/.exec(directoryUrl)
|
||||
if (!idMatch) {
|
||||
const fallback = FEED_URL_RE.exec(html)
|
||||
return fallback ? fallback[1] : null
|
||||
}
|
||||
|
||||
const adamIdx = html.search(new RegExp(`"adamId"\\s*:\\s*"${idMatch[1]}"`))
|
||||
if (adamIdx < 0) return null
|
||||
|
||||
const fromAdam = new RegExp(FEED_URL_RE.source, "g")
|
||||
fromAdam.lastIndex = adamIdx
|
||||
const match = fromAdam.exec(html)
|
||||
return match ? match[1] : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a delisted show's RSS feed from its Apple Podcasts page.
|
||||
* Returns null on network failure or when the page has no resolvable feed.
|
||||
*/
|
||||
export const resolveItunesFeedUrl = async (
|
||||
directoryUrl: string,
|
||||
): Promise<string | null> => {
|
||||
try {
|
||||
const response = await fetch(directoryUrl, {
|
||||
headers: { "User-Agent": "PodTUI/1.0" },
|
||||
})
|
||||
if (!response.ok) return null
|
||||
return extractFeedUrlFromPage(await response.text(), directoryUrl)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -14,11 +14,13 @@ type ItunesResult = {
|
||||
collectionId?: number
|
||||
collectionName?: string
|
||||
artistName?: string
|
||||
feedUrl?: string
|
||||
/** Null for shows delisted from Apple Podcasts (directory stub records). */
|
||||
feedUrl?: string | null
|
||||
artworkUrl100?: string
|
||||
artworkUrl600?: string
|
||||
primaryGenreName?: string
|
||||
releaseDate?: string
|
||||
collectionViewUrl?: string
|
||||
}
|
||||
|
||||
type ItunesResponse = {
|
||||
@@ -41,8 +43,8 @@ const buildItunesUrl = (query: string, source: PodcastSource) => {
|
||||
return url.toString()
|
||||
}
|
||||
|
||||
const mapItunesResult = (result: ItunesResult, source: PodcastSource): Podcast | null => {
|
||||
if (!result.collectionName || !result.feedUrl) return null
|
||||
export const mapItunesResult = (result: ItunesResult, source: PodcastSource): Podcast | null => {
|
||||
if (!result.collectionName) return null
|
||||
|
||||
const id = result.collectionId
|
||||
? `itunes-${result.collectionId}`
|
||||
@@ -52,11 +54,18 @@ const mapItunesResult = (result: ItunesResult, source: PodcastSource): Podcast |
|
||||
if (result.artistName) descriptionParts.push(`by ${result.artistName}`)
|
||||
if (result.primaryGenreName) descriptionParts.push(result.primaryGenreName)
|
||||
|
||||
// Shows delisted from Apple Podcasts (e.g. The Daily Wire's shows) come back
|
||||
// as metadata-only stub records with feedUrl null. Keep the stub so the show
|
||||
// stays findable; the real feed is resolved from the directory page at
|
||||
// subscribe time (see itunes-feed-resolver).
|
||||
const feedUrl = result.feedUrl ?? ""
|
||||
|
||||
return {
|
||||
id,
|
||||
title: result.collectionName,
|
||||
description: descriptionParts.join(" • "),
|
||||
feedUrl: result.feedUrl,
|
||||
feedUrl,
|
||||
directoryUrl: feedUrl ? undefined : result.collectionViewUrl,
|
||||
author: result.artistName,
|
||||
categories: result.primaryGenreName ? [result.primaryGenreName] : undefined,
|
||||
coverUrl: result.artworkUrl600 || result.artworkUrl100,
|
||||
|
||||
Reference in New Issue
Block a user