zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resolve external embed previews and keep their real MIME types

Browse Source
This commit is contained in:
Fisto
2026-09-16 12:52:44 +02:00
parent 7fa6118b6f
commit 1b5fcfed2f
11 changed files with 313 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/download.ts
+20 -2
View File
@@ -59,9 +59,25 @@ export interface DownloadOutcome {
path?: string;
hash?: string;
bytes?: number;
/** File extension of the stored bytes, when known. */
ext?: string;
error?: string;
}
const PREVIEW_EXTENSIONS: Record<string, string> = {
"image/webp": "webp",
"image/png": "png",
"image/jpeg": "jpg",
"image/gif": "gif",
"image/avif": "avif"
};
/** Extension for preview bytes from the response type, defaulting to webp. */
export function previewExtension(contentType: string | null): string {
const type = (contentType ?? "").split(";")[0].trim().toLowerCase();
return PREVIEW_EXTENSIONS[type] ?? "webp";
}
/** 4xx that will not improve on retry: the signed URL expired or the file is gone. */
const SKIP_STATUS = new Set([400, 401, 403, 404, 410]);
@@ -214,11 +230,13 @@ export async function downloadPreview(
return { id: media.id, state: "skipped", error: "too large" };
}
// Keep the real type: provider thumbnails are often JPEG or PNG, not WebP.
const ext = previewExtension(response.headers.get("content-type"));
const hash = createHash("sha256").update(buffer).digest("hex");
const file = `${hash}.webp`;
const file = `${hash}.${ext}`;
writeFileSync(join(dir, file), buffer);
return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength };
return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength, ext };
} catch (error) {
return {
id: media.id,
media-scrape/src/embed-preview.ts
+114
View File
@@ -0,0 +1,114 @@
/*
* Resolve a preview image for embeds that carry none in Discord's data.
* Provider oEmbed/API first, then the page's og:image / twitter:image.
*/
import type { DownloadOptions } from "./download.js";
import type { MediaRow } from "./store.js";
const USER_AGENT = "veskslop-media/0.1";
const MAX_HTML_BYTES = 512 * 1024;
const TIMEOUT_MS = 10_000;
function fetchOf(options: DownloadOptions): typeof fetch {
return options.fetchImpl ?? fetch;
}
async function getJson(url: string, options: DownloadOptions): Promise<any | null> {
try {
const response = await fetchOf(options)(url, {
headers: { "user-agent": USER_AGENT, accept: "application/json" },
signal: AbortSignal.timeout(TIMEOUT_MS)
});
if (!response.ok) return null;
return await response.json();
} catch {
return null;
}
}
async function getText(url: string, options: DownloadOptions): Promise<string | null> {
try {
const response = await fetchOf(options)(url, {
headers: { "user-agent": USER_AGENT, accept: "text/html" },
signal: AbortSignal.timeout(TIMEOUT_MS)
});
if (!response.ok) return null;
return (await response.text()).slice(0, MAX_HTML_BYTES);
} catch {
return null;
}
}
/** First og:image or twitter:image in the page head, either attribute order. */
export function ogImageFromHtml(html: string): string | null {
for (const pattern of [
/<meta[^>]+(?:property|name)=["']og:image["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["']og:image["']/i,
/<meta[^>]+(?:property|name)=["']twitter:image["'][^>]+content=["']([^"']+)["']/i,
/<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["']twitter:image["']/i
]) {
const match = pattern.exec(html);
if (match?.[1]) return match[1];
}
return null;
}
/**
* A preview URL for an embed. Uses the stored thumbnail when present, otherwise asks the
* provider (oEmbed, fxtwitter) and finally falls back to the page's og:image.
*/
export async function resolveEmbedPreview(
media: Pick<MediaRow, "url" | "source" | "preview_url">,
options: DownloadOptions = {}
): Promise<string | null> {
if (media.source !== "embed") return null;
if (media.preview_url) return media.preview_url;
let parsed: URL;
try {
parsed = new URL(media.url);
} catch {
return null;
}
const host = parsed.hostname.replace(/^www\./, "").toLowerCase();
if (host === "youtube.com" || host === "m.youtube.com" || host === "youtu.be") {
const data = await getJson(
`https://www.youtube.com/oembed?url=${encodeURIComponent(media.url)}&format=json`,
options
);
if (typeof data?.thumbnail_url === "string") return data.thumbnail_url;
}
if (host === "vimeo.com") {
const data = await getJson(
`https://vimeo.com/api/oembed.json?url=${encodeURIComponent(media.url)}`,
options
);
if (typeof data?.thumbnail_url === "string") return data.thumbnail_url;
}
if (host === "tiktok.com") {
const data = await getJson(
`https://www.tiktok.com/oembed?url=${encodeURIComponent(media.url)}`,
options
);
if (typeof data?.thumbnail_url === "string") return data.thumbnail_url;
}
if (host === "fxtwitter.com" || host === "vxtwitter.com" || host === "twitter.com" || host === "x.com") {
const match = /\/status\/(\d+)/.exec(parsed.pathname);
if (match) {
const data = await getJson(`https://api.fxtwitter.com/status/${match[1]}`, options);
const video = data?.tweet?.media?.videos?.[0]?.thumbnail_url;
const photo = data?.tweet?.media?.photos?.[0]?.url;
if (typeof video === "string") return video;
if (typeof photo === "string") return photo;
}
}
const html = await getText(media.url, options);
return html ? ogImageFromHtml(html) : null;
}
media-scrape/src/engine.ts
+25 -4
View File
@@ -13,6 +13,7 @@ import {
videoPoster,
type DownloadOptions
} from "./download.js";
import { resolveEmbedPreview } from "./embed-preview.js";
import {
createScrapeDirs,
listScrapes,
@@ -358,18 +359,38 @@ export class ScrapeEngine {
return;
}
const outcome = await downloadPreview(row, join(dir, "thumbs"), options);
// Some providers (fxtwitter, vxReddit, ...) ship no thumbnail in Discord's data;
// resolve one from the provider or the page before caching it.
let previewRow = row;
if (row.source === "embed" && !row.preview_url) {
const resolved = await resolveEmbedPreview(row, options);
if (!resolved) {
store.markPreviewError(row.id, "no preview source");
this.persist(dir, meta);
await this.sleep(downloadGapMs(options));
continue;
}
previewRow = { ...row, preview_url: resolved };
}
const outcome = await downloadPreview(previewRow, join(dir, "thumbs"), options);
if (outcome.state === "downloaded" && outcome.hash) {
store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0 });
const ext = outcome.ext ?? "webp";
store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0, ext });
meta.previewed = (meta.previewed ?? 0) + 1;
// Embed thumbnails also land in the custom destination, so the download has a
// viewable poster for embeds while the media tab keeps its local copy.
if (options.destinationDir && row.source === "embed") {
const target = join(options.destinationDir, meta.id, "thumbnails", `${outcome.hash}.webp`);
const target = join(
options.destinationDir,
meta.id,
"thumbnails",
`${outcome.hash}.${ext}`
);
mkdirSync(dirname(target), { recursive: true });
copyFileSync(join(dir, "thumbs", `${outcome.hash}.webp`), target);
copyFileSync(join(dir, "thumbs", `${outcome.hash}.${ext}`), target);
}
} else {
store.markPreviewError(row.id, outcome.error ?? outcome.state);
media-scrape/src/index.ts
+1
View File
@@ -4,6 +4,7 @@
*/
export { extractMedia, type ExtractedMedia } from "./extract.js";
export { ogImageFromHtml, resolveEmbedPreview } from "./embed-preview.js";
export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
export {
media-scrape/src/store.ts
+8 -3
View File
@@ -133,6 +133,7 @@ export interface MediaRow {
preview_url: string | null;
file_hash: string | null;
preview_hash: string | null;
preview_ext: string | null;
file_bytes: number | null;
preview_bytes: number | null;
file_path: string | null;
@@ -169,6 +170,7 @@ CREATE TABLE IF NOT EXISTS media (
preview_url TEXT,
file_hash TEXT,
preview_hash TEXT,
preview_ext TEXT,
file_bytes INTEGER,
preview_bytes INTEGER,
file_path TEXT,
@@ -212,6 +214,7 @@ export class ScrapeStore {
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
if (!names.has("preview_ext")) this.db.exec("ALTER TABLE media ADD COLUMN preview_ext TEXT");
if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
}
@@ -347,10 +350,12 @@ export class ScrapeStore {
.all(limit) as unknown as MediaRow[];
}
markPreview(id: number, preview: { hash: string; bytes: number }): void {
markPreview(id: number, preview: { hash: string; bytes: number; ext?: string }): void {
this.db
.prepare("UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_error = NULL WHERE id = ?")
.run(preview.hash, preview.bytes, id);
.prepare(
"UPDATE media SET preview_hash = ?, preview_bytes = ?, preview_ext = ?, preview_error = NULL WHERE id = ?"
)
.run(preview.hash, preview.bytes, preview.ext ?? "webp", id);
}
markPreviewError(id: number, error: string): void {