zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cache embed thumbnails in download mode

Browse Source
This commit is contained in:
Fisto
2026-09-16 11:17:37 +02:00
parent 902646224f
commit 1fe204b91e
7 changed files with 161 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/download.ts
+34 -13
View File
@@ -19,6 +19,8 @@ export interface DownloadOptions {
maxRetries?: number;
/** Preview size cap. Default 5 MiB. */
previewMaxBytes?: number;
/** Cache third-party embed thumbnails. Download mode only. */
embedPreviews?: boolean;
sleep?: (ms: number) => Promise<void>;
random?: () => number;
log?: (line: string) => void;
@@ -125,20 +127,30 @@ export async function downloadOne(
const DISCORD_HOSTS = new Set(["cdn.discordapp.com", "media.discordapp.net"]);
/**
* A small WebP preview URL for a Discord-hosted image or GIF.
* Third-party embeds return null: they are never fetched.
* The preview target for a media row.
*
* Attachments and stickers: their Discord-hosted URL, resized to a small WebP.
* Embeds: the stored thumbnail or image URL, used as-is (Discord-hosted ones are still resized).
* Returns null when the row has no preview source.
*/
export function previewUrlFor(media: { url: string; kind: string }): string | null {
if (media.kind !== "image" && media.kind !== "gif") return null;
export function previewUrlFor(media: {
url: string;
kind: string;
previewUrl?: string | null;
}): string | null {
const candidate =
media.previewUrl ??
(media.kind === "image" || media.kind === "gif" ? media.url : null);
if (!candidate) return null;
try {
const url = new URL(media.url);
if (!DISCORD_HOSTS.has(url.hostname)) return null;
url.hostname = "media.discordapp.net";
url.searchParams.set("width", "320");
url.searchParams.set("height", "320");
url.searchParams.set("format", "webp");
const url = new URL(candidate);
if (DISCORD_HOSTS.has(url.hostname)) {
url.hostname = "media.discordapp.net";
url.searchParams.set("width", "320");
url.searchParams.set("height", "320");
url.searchParams.set("format", "webp");
}
return url.toString();
} catch {
return null;
@@ -147,11 +159,20 @@ export function previewUrlFor(media: { url: string; kind: string }): string | nu
/** Fetch a small preview into `dir`. One attempt; a failure never fails the scrape. */
export async function downloadPreview(
media: Pick<MediaRow, "id" | "url" | "kind">,
media: Pick<MediaRow, "id" | "url" | "kind" | "source" | "preview_url">,
dir: string,
options: DownloadOptions = {}
): Promise<DownloadOutcome> {
const target = previewUrlFor(media);
// Third-party thumbnails only in download mode; the full embed is never fetched.
if (media.source === "embed" && options.embedPreviews !== true) {
return { id: media.id, state: "skipped", error: "embed previews off" };
}
const target = previewUrlFor({
url: media.url,
kind: media.kind,
previewUrl: media.preview_url
});
if (!target) return { id: media.id, state: "skipped", error: "no preview url" };
const fetchImpl = options.fetchImpl ?? fetch;
media-scrape/src/engine.ts
+21 -8
View File
@@ -220,12 +220,24 @@ export class ScrapeEngine {
await this.sleep(this.gapMs());
}
if (meta.state === "done" && meta.mode === "download") {
await this.runDownloads(store, dir, meta);
}
if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
// Keep the job running until post-processing finishes, so a client never
// treats a half-archived scrape as complete.
meta.state = "running";
this.persist(dir, meta);
if (meta.state === "done" && this.options.previews !== false) {
await this.runPreviews(store, dir, meta);
if (meta.mode === "download") {
await this.runDownloads(store, dir, meta);
}
if (this.options.previews !== false && !this.stopRequested.has(meta.id)) {
await this.runPreviews(store, dir, meta);
}
if (meta.state === "running") {
meta.state = "done";
}
this.persist(dir, meta);
}
} catch (error) {
meta.state = "failed";
@@ -271,10 +283,11 @@ export class ScrapeEngine {
}
}
/** Cache small previews for Discord-hosted images. Runs in both modes. */
/** Cache small previews. In download mode this includes embed thumbnails. */
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const options = this.options.download ?? {};
const pending = store.pendingPreviews();
const embedPreviews = meta.mode === "download";
const options = { ...(this.options.download ?? {}), embedPreviews };
const pending = store.pendingPreviews(5000, embedPreviews);
for (const row of pending) {
if (this.stopRequested.has(meta.id)) {
media-scrape/src/store.ts
+11 -3
View File
@@ -294,11 +294,19 @@ export class ScrapeStore {
this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id);
}
/** Image and GIF rows that still need a cached preview. */
pendingPreviews(limit = 5000): MediaRow[] {
/**
* Rows that still need a cached preview. Attachment/sticker images and GIFs always;
* embed thumbnails only when `includeEmbeds` is set (download mode).
*/
pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
const local = "(kind IN ('image', 'gif') AND (source IS NULL OR source != 'embed'))";
const condition = includeEmbeds
? `(${local} OR (source = 'embed' AND preview_url IS NOT NULL))`
: local;
return this.db
.prepare(
"SELECT * FROM media WHERE (kind = 'image' OR kind = 'gif') AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?"
`SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?`
)
.all(limit) as unknown as MediaRow[];
}
media-scrape/test/run.ts
+76 -11
View File
@@ -351,7 +351,7 @@ void (async () => {
provider: "YouTube",
videoUrl: "https://www.youtube.com/embed/x",
imageUrl: null,
thumbnailUrl: null
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
}
]
})
@@ -359,6 +359,7 @@ void (async () => {
}
]);
const fetched: string[] = [];
const engine = new ScrapeEngine({
rootDir: root,
pageSize: 2,
@@ -367,10 +368,15 @@ void (async () => {
sleep: async () => {},
random: () => 0,
log: () => {},
previews: false,
previews: true,
download: {
fetchImpl: async () =>
new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
fetchImpl: async (input: any) => {
fetched.push(String(input));
return new Response("png-bytes", {
status: 200,
headers: { "content-type": "image/png" }
});
},
minGapMs: 0,
maxGapMs: 0,
sleep: async () => {}
@@ -383,6 +389,9 @@ void (async () => {
const meta = engine.get(started.id);
assert.equal(meta?.state, "done");
assert.equal(meta?.downloaded, 1);
assert.equal(meta?.previewed, 2);
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
assert.equal(fetched.some(url => url.includes("i.ytimg.com")), true);
const store = new ScrapeStore(join(root, started.id));
const row = store.queryMedia({ kind: "image" })[0];
@@ -426,13 +435,20 @@ void (async () => {
store.close();
});
await test("preview urls are Discord-only resizes", () => {
await test("preview urls resize Discord and pass embed thumbnails through", () => {
assert.equal(
previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }),
"https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp"
);
assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null);
assert.equal(previewUrlFor({ url: "https://i.ytimg.com/vi/x/hq.jpg", kind: "image" }), null);
assert.equal(
previewUrlFor({
url: "https://www.youtube.com/watch?v=x",
kind: "video",
previewUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
}),
"https://i.ytimg.com/vi/x/hqdefault.jpg"
);
});
await test("preview downloads are written by hash", async () => {
@@ -440,7 +456,13 @@ void (async () => {
mkdirSync(dir, { recursive: true });
const outcome = await downloadPreview(
{ id: 9, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" },
{
id: 9,
url: "https://cdn.discordapp.com/attachments/1/2/pic.png",
kind: "image",
source: "attachment",
preview_url: null
},
dir,
{
fetchImpl: async () =>
@@ -453,17 +475,57 @@ void (async () => {
assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true);
});
await test("engine caches previews in links mode", async () => {
await test("embed thumbnails download only when enabled", async () => {
const dir = join(root, "preview-embed");
mkdirSync(dir, { recursive: true });
const media = {
id: 11,
url: "https://www.youtube.com/watch?v=x",
kind: "video" as const,
source: "embed",
preview_url: "https://i.ytimg.com/vi/x/hqdefault.jpg"
};
const calls: string[] = [];
const fetchImpl = async (input: any) => {
calls.push(String(input));
return new Response("thumb", { status: 200, headers: { "content-type": "image/webp" } });
};
const off = await downloadPreview(media, dir, { fetchImpl });
assert.equal(off.state, "skipped");
assert.equal(calls.length, 0);
const on = await downloadPreview(media, dir, { fetchImpl, embedPreviews: true });
assert.equal(on.state, "downloaded");
assert.equal(calls[0], "https://i.ytimg.com/vi/x/hqdefault.jpg");
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
});
await test("engine caches previews in links mode and skips embeds", async () => {
const backend = new FakeBackend([
{
messages: [
message("600", {
attachments: [attachment({ filename: "prev.png", contentType: "image/png" })]
attachments: [attachment({ filename: "prev.png", contentType: "image/png" })],
embeds: [
{
type: "video",
url: "https://www.youtube.com/watch?v=x",
title: null,
provider: "YouTube",
videoUrl: "https://www.youtube.com/embed/x",
imageUrl: null,
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
}
]
})
]
}
]);
const fetched: string[] = [];
const engine = new ScrapeEngine({
rootDir: root,
pageSize: 2,
@@ -474,8 +536,10 @@ void (async () => {
log: () => {},
previews: true,
download: {
fetchImpl: async () =>
new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }),
fetchImpl: async (input: any) => {
fetched.push(String(input));
return new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } });
},
minGapMs: 0,
maxGapMs: 0,
sleep: async () => {}
@@ -488,6 +552,7 @@ void (async () => {
const meta = engine.get(started.id);
assert.equal(meta?.state, "done");
assert.equal(meta?.previewed, 1);
assert.equal(fetched.some(url => url.includes("ytimg.com")), false);
const store = new ScrapeStore(join(root, started.id));
const row = store.queryMedia({ kind: "image" })[0];