Cache embed thumbnails in download mode
This commit is contained in:
Fisto
@@ -40,6 +40,11 @@ mobile draft at 390px.
|
||||
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
|
||||
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
|
||||
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
|
||||
- [x] C13 Embed thumbnails in download mode: embed rows cache their stored thumbnail URL;
|
||||
the full third-party media is never fetched. Live run `...-20260916091458` on the test
|
||||
channel: 6 files downloaded, 11 previews cached (YouTube, Reddit, Tenor, TikTok, X). Also
|
||||
fixed the state transition: the job stays `running` through download and preview passes so
|
||||
clients do not see `done` early.
|
||||
- [x] C12 Media tab filters and full-window layout: kind chips, source filter
|
||||
(attachment/sticker/embed), source-channel filter, and the view now fills the window below the
|
||||
header and call strip. API/MCP gained `source` and `channelId` on `query_media`. Smoke-tested
|
||||
|
||||
@@ -101,14 +101,18 @@ Downloads run after a finished walk when the scrape used `mode: "download"`:
|
||||
(`source != 'embed'`).
|
||||
- Progress counters: `downloaded`, `downloadErrors`.
|
||||
|
||||
Previews run after every walk, in both modes:
|
||||
Previews run after every walk:
|
||||
|
||||
- Only Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`) are
|
||||
transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`).
|
||||
Third-party embeds are skipped.
|
||||
- In both modes, Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`)
|
||||
are transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`).
|
||||
- In `download` mode only, embed rows also get their stored `preview_url` (YouTube, X, TikTok
|
||||
thumbnails, and so on) cached. The embed's full media URL is never fetched; embed rows are
|
||||
excluded from `pendingDownloads`.
|
||||
- One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape.
|
||||
- Stored by hash in `thumbs/<sha256>.webp`; `preview_hash` and `preview_bytes` land on the row, and
|
||||
the counter is `previewed`. Previews count toward the storage limit.
|
||||
- The job stays `running` through the download and preview passes, so a client never sees `done`
|
||||
on a half-archived scrape.
|
||||
|
||||
## Job states
|
||||
|
||||
|
||||
+6
-8
@@ -80,14 +80,12 @@ media-scrape/db/
|
||||
- `download` fetches attachment bytes into `media/`, one file at a time, hash-addressed. Embed rows
|
||||
are never download targets. The Media tab prefers the local file when one exists, so the archive
|
||||
survives CDN URL expiry.
|
||||
- Discord-hosted image and GIF previews are cached small (WebP, 320px) in `thumbs/` after every
|
||||
walk, in both modes, so the grid does not depend on expiring CDN links. Third-party embed
|
||||
thumbnails are never fetched.
|
||||
- Third-party embeds are **never downloaded**: YouTube, Instagram, X, TikTok, and every other
|
||||
embed stays a link with metadata.
|
||||
- Discord-hosted image and sticker previews are cached small (`thumbs/`, WebP, via the CDN
|
||||
resizer) in both modes, so the Media grid does not depend on expiring CDN links. Previews count
|
||||
toward the storage limit.
|
||||
- Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every
|
||||
other embed stays a link. In `download` mode their thumbnail is cached (below); the video itself
|
||||
is never fetched.
|
||||
- Previews are cached small (WebP, 320px) in `thumbs/` after every walk. In both modes this covers
|
||||
Discord-hosted images and GIFs; in `download` mode it also covers embed thumbnails, so the grid
|
||||
does not depend on expiring CDN links. Previews count toward the storage limit.
|
||||
- Discord attachment URLs are signed and expire in about a day. In `links` mode an expired URL is
|
||||
refreshed by re-fetching its message; in `download` mode the bytes are caught before expiry.
|
||||
|
||||
|
||||
@@ -19,6 +19,8 @@ export interface DownloadOptions {
|
||||
maxRetries?: number;
|
||||
/** Preview size cap. Default 5 MiB. */
|
||||
previewMaxBytes?: number;
|
||||
/** Cache third-party embed thumbnails. Download mode only. */
|
||||
embedPreviews?: boolean;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
random?: () => number;
|
||||
log?: (line: string) => void;
|
||||
@@ -125,20 +127,30 @@ export async function downloadOne(
|
||||
const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]);
|
||||
|
||||
/**
|
||||
* A small WebP preview URL for a Discord-hosted image or GIF.
|
||||
* Third-party embeds return null: they are never fetched.
|
||||
* The preview target for a media row.
|
||||
*
|
||||
* Attachments and stickers: their Discord-hosted URL, resized to a small WebP.
|
||||
* Embeds: the stored thumbnail or image URL, used as-is (Discord-hosted ones are still resized).
|
||||
* Returns null when the row has no preview source.
|
||||
*/
|
||||
export function previewUrlFor(media: { url: string; kind: string }): string | null {
|
||||
if (media.kind !== "image" && media.kind !== "gif") return null;
|
||||
export function previewUrlFor(media: {
|
||||
url: string;
|
||||
kind: string;
|
||||
previewUrl?: string | null;
|
||||
}): string | null {
|
||||
const candidate =
|
||||
media.previewUrl ??
|
||||
(media.kind === "image" || media.kind === "gif" ? media.url : null);
|
||||
if (!candidate) return null;
|
||||
|
||||
try {
|
||||
const url = new URL(media.url);
|
||||
if (!DISCORD_HOSTS.has(url.hostname)) return null;
|
||||
|
||||
url.hostname = "media.discordapp.net";
|
||||
url.searchParams.set("width", "320");
|
||||
url.searchParams.set("height", "320");
|
||||
url.searchParams.set("format", "webp");
|
||||
const url = new URL(candidate);
|
||||
if (DISCORD_HOSTS.has(url.hostname)) {
|
||||
url.hostname = "media.discordapp.net";
|
||||
url.searchParams.set("width", "320");
|
||||
url.searchParams.set("height", "320");
|
||||
url.searchParams.set("format", "webp");
|
||||
}
|
||||
return url.toString();
|
||||
} catch {
|
||||
return null;
|
||||
@@ -147,11 +159,20 @@ export function previewUrlFor(media: { url: string; kind: string }): string | nu
|
||||
|
||||
/** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */
|
||||
export async function downloadPreview(
|
||||
media: Pick<MediaRow, "id" | "url" | "kind">,
|
||||
media: Pick<MediaRow, "id" | "url" | "kind" | "source" | "preview_url">,
|
||||
dir: string,
|
||||
options: DownloadOptions = {}
|
||||
): Promise<DownloadOutcome> {
|
||||
const target = previewUrlFor(media);
|
||||
// Third-party thumbnails only in download mode; the full embed is never fetched.
|
||||
if (media.source === "embed" && options.embedPreviews !== true) {
|
||||
return { id: media.id, state: "skipped", error: "embed previews off" };
|
||||
}
|
||||
|
||||
const target = previewUrlFor({
|
||||
url: media.url,
|
||||
kind: media.kind,
|
||||
previewUrl: media.preview_url
|
||||
});
|
||||
if (!target) return { id: media.id, state: "skipped", error: "no preview url" };
|
||||
|
||||
const fetchImpl = options.fetchImpl ?? fetch;
|
||||
|
||||
@@ -220,12 +220,24 @@ export class ScrapeEngine {
|
||||
await this.sleep(this.gapMs());
|
||||
}
|
||||
|
||||
if (meta.state === "done" && meta.mode === "download") {
|
||||
await this.runDownloads(store, dir, meta);
|
||||
}
|
||||
if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
|
||||
// Keep the job running until post-processing finishes, so a client never
|
||||
// treats a half-archived scrape as complete.
|
||||
meta.state = "running";
|
||||
this.persist(dir, meta);
|
||||
|
||||
if (meta.state === "done" && this.options.previews !== false) {
|
||||
await this.runPreviews(store, dir, meta);
|
||||
if (meta.mode === "download") {
|
||||
await this.runDownloads(store, dir, meta);
|
||||
}
|
||||
|
||||
if (this.options.previews !== false && !this.stopRequested.has(meta.id)) {
|
||||
await this.runPreviews(store, dir, meta);
|
||||
}
|
||||
|
||||
if (meta.state === "running") {
|
||||
meta.state = "done";
|
||||
}
|
||||
this.persist(dir, meta);
|
||||
}
|
||||
} catch (error) {
|
||||
meta.state = "failed";
|
||||
@@ -271,10 +283,11 @@ export class ScrapeEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/** Cache small previews for Discord-hosted images. Runs in both modes. */
|
||||
/** Cache small previews. In download mode this includes embed thumbnails. */
|
||||
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
|
||||
const options = this.options.download ?? {};
|
||||
const pending = store.pendingPreviews();
|
||||
const embedPreviews = meta.mode === "download";
|
||||
const options = { ...(this.options.download ?? {}), embedPreviews };
|
||||
const pending = store.pendingPreviews(5000, embedPreviews);
|
||||
|
||||
for (const row of pending) {
|
||||
if (this.stopRequested.has(meta.id)) {
|
||||
|
||||
@@ -294,11 +294,19 @@ export class ScrapeStore {
|
||||
this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id);
|
||||
}
|
||||
|
||||
/** Image and GIF rows that still need a cached preview. */
|
||||
pendingPreviews(limit = 5000): MediaRow[] {
|
||||
/**
|
||||
* Rows that still need a cached preview. Attachment/sticker images and GIFs always;
|
||||
* embed thumbnails only when `includeEmbeds` is set (download mode).
|
||||
*/
|
||||
pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
|
||||
const local = "(kind IN ('image', 'gif') AND (source IS NULL OR source != 'embed'))";
|
||||
const condition = includeEmbeds
|
||||
? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))`
|
||||
: local;
|
||||
|
||||
return this.db
|
||||
.prepare(
|
||||
"SELECT * FROM media WHERE (kind = 'image' OR kind = 'gif') AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?"
|
||||
`SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?`
|
||||
)
|
||||
.all(limit) as unknown as MediaRow[];
|
||||
}
|
||||
|
||||
+76
-11
@@ -351,7 +351,7 @@ void (async () => {
|
||||
provider: "YouTube",
|
||||
videoUrl: "https://www.youtube.com/embed/x",
|
||||
imageUrl: null,
|
||||
thumbnailUrl: null
|
||||
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
}
|
||||
]
|
||||
})
|
||||
@@ -359,6 +359,7 @@ void (async () => {
|
||||
}
|
||||
]);
|
||||
|
||||
const fetched: string[] = [];
|
||||
const engine = new ScrapeEngine({
|
||||
rootDir: root,
|
||||
pageSize: 2,
|
||||
@@ -367,10 +368,15 @@ void (async () => {
|
||||
sleep: async () => {},
|
||||
random: () => 0,
|
||||
log: () => {},
|
||||
previews: false,
|
||||
previews: true,
|
||||
download: {
|
||||
fetchImpl: async () =>
|
||||
new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
|
||||
fetchImpl: async (input: any) => {
|
||||
fetched.push(String(input));
|
||||
return new Response("png-bytes", {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" }
|
||||
});
|
||||
},
|
||||
minGapMs: 0,
|
||||
maxGapMs: 0,
|
||||
sleep: async () => {}
|
||||
@@ -383,6 +389,9 @@ void (async () => {
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "done");
|
||||
assert.equal(meta?.downloaded, 1);
|
||||
assert.equal(meta?.previewed, 2);
|
||||
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
|
||||
assert.equal(fetched.some(url => url.includes("i.ytimg.com")), true);
|
||||
|
||||
const store = new ScrapeStore(join(root, started.id));
|
||||
const row = store.queryMedia({ kind: "image" })[0];
|
||||
@@ -426,13 +435,20 @@ void (async () => {
|
||||
store.close();
|
||||
});
|
||||
|
||||
await test("preview urls are Discord-only resizes", () => {
|
||||
await test("preview urls resize Discord and pass embed thumbnails through", () => {
|
||||
assert.equal(
|
||||
previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }),
|
||||
"https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp"
|
||||
);
|
||||
assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null);
|
||||
assert.equal(previewUrlFor({ url: "https://i.ytimg.com/vi/x/hq.jpg", kind: "image" }), null);
|
||||
assert.equal(
|
||||
previewUrlFor({
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
kind: "video",
|
||||
previewUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
}),
|
||||
"https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
);
|
||||
});
|
||||
|
||||
await test("preview downloads are written by hash", async () => {
|
||||
@@ -440,7 +456,13 @@ void (async () => {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
const outcome = await downloadPreview(
|
||||
{ id: 9, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" },
|
||||
{
|
||||
id: 9,
|
||||
url: "https://cdn.discordapp.com/attachments/1/2/pic.png",
|
||||
kind: "image",
|
||||
source: "attachment",
|
||||
preview_url: null
|
||||
},
|
||||
dir,
|
||||
{
|
||||
fetchImpl: async () =>
|
||||
@@ -453,17 +475,57 @@ void (async () => {
|
||||
assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true);
|
||||
});
|
||||
|
||||
await test("engine caches previews in links mode", async () => {
|
||||
await test("embed thumbnails download only when enabled", async () => {
|
||||
const dir = join(root, "preview-embed");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
const media = {
|
||||
id: 11,
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
kind: "video" as const,
|
||||
source: "embed",
|
||||
preview_url: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const fetchImpl = async (input: any) => {
|
||||
calls.push(String(input));
|
||||
return new Response("thumb", { status: 200, headers: { "content-type": "image/webp" } });
|
||||
};
|
||||
|
||||
const off = await downloadPreview(media, dir, { fetchImpl });
|
||||
assert.equal(off.state, "skipped");
|
||||
assert.equal(calls.length, 0);
|
||||
|
||||
const on = await downloadPreview(media, dir, { fetchImpl, embedPreviews: true });
|
||||
assert.equal(on.state, "downloaded");
|
||||
assert.equal(calls[0], "https://i.ytimg.com/vi/x/hqdefault.jpg");
|
||||
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
|
||||
});
|
||||
|
||||
await test("engine caches previews in links mode and skips embeds", async () => {
|
||||
const backend = new FakeBackend([
|
||||
{
|
||||
messages: [
|
||||
message("600", {
|
||||
attachments: [attachment({ filename: "prev.png", contentType: "image/png" })]
|
||||
attachments: [attachment({ filename: "prev.png", contentType: "image/png" })],
|
||||
embeds: [
|
||||
{
|
||||
type: "video",
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
title: null,
|
||||
provider: "YouTube",
|
||||
videoUrl: "https://www.youtube.com/embed/x",
|
||||
imageUrl: null,
|
||||
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
}
|
||||
]
|
||||
})
|
||||
]
|
||||
}
|
||||
]);
|
||||
|
||||
const fetched: string[] = [];
|
||||
|
||||
const engine = new ScrapeEngine({
|
||||
rootDir: root,
|
||||
pageSize: 2,
|
||||
@@ -474,8 +536,10 @@ void (async () => {
|
||||
log: () => {},
|
||||
previews: true,
|
||||
download: {
|
||||
fetchImpl: async () =>
|
||||
new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }),
|
||||
fetchImpl: async (input: any) => {
|
||||
fetched.push(String(input));
|
||||
return new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } });
|
||||
},
|
||||
minGapMs: 0,
|
||||
maxGapMs: 0,
|
||||
sleep: async () => {}
|
||||
@@ -488,6 +552,7 @@ void (async () => {
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "done");
|
||||
assert.equal(meta?.previewed, 1);
|
||||
assert.equal(fetched.some(url => url.includes("ytimg.com")), false);
|
||||
|
||||
const store = new ScrapeStore(join(root, started.id));
|
||||
const row = store.queryMedia({ kind: "image" })[0];
|
||||
|
||||
Reference in New Issue
Block a user