zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add links-mode embed thumbnails, item metadata, and media filters

Browse Source
This commit is contained in:
Fisto
2026-09-16 11:44:09 +02:00
parent 1fe204b91e
commit 973b9cff92
23 changed files with 863 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/slopbot/backend.ts
+14 -1
View File
@@ -37,11 +37,22 @@ function mapAttachments(message: any): CoreAttachment[] {
}));
}
/** Dimensions of the first embed part that has them: video, image, then thumbnail. */
function embedDataSize(data: any) {
for (const part of [data?.video, data?.image, data?.thumbnail]) {
if (part && Number.isFinite(part.width) && Number.isFinite(part.height)) {
return { width: part.width, height: part.height };
}
}
return { width: null, height: null };
}
function mapEmbeds(message: any): CoreEmbed[] {
const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null);
return collectionValues(message?.embeds).map((embed: any) => {
const data = embed?.data ?? embed ?? {};
const size = embedDataSize(data);
return {
type: data.type ?? null,
url: data.url ?? null,
@@ -49,7 +60,9 @@ function mapEmbeds(message: any): CoreEmbed[] {
provider: data.provider?.name ?? null,
videoUrl: assetUrl(data.video),
imageUrl: assetUrl(data.image),
thumbnailUrl: assetUrl(data.thumbnail)
thumbnailUrl: assetUrl(data.thumbnail),
width: size.width,
height: size.height
};
});
}
media-scrape/src/download.ts
+25
View File
@@ -3,11 +3,36 @@
* Rules: docs/media-scraper.md. Third-party embeds are never targets here.
*/
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import { promisify } from "node:util";
import type { MediaRow } from "./store.js";
const execFileAsync = promisify(execFile);
/** Read pixel dimensions with ffprobe. Null when the tool is missing or the file is not media. */
export async function probeDimensions(
path: string,
ffprobe = process.env.VESKSLOP_FFPROBE ?? "ffprobe"
): Promise<{ width: number; height: number } | null> {
try {
const { stdout } = await execFileAsync(
ffprobe,
["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "json", path],
{ timeout: 15_000 }
);
const stream = JSON.parse(stdout)?.streams?.[0];
if (Number.isFinite(stream?.width) && Number.isFinite(stream?.height)) {
return { width: stream.width, height: stream.height };
}
} catch {
// missing ffprobe, a timeout, or a non-visual file
}
return null;
}
export interface DownloadOptions {
fetchImpl?: typeof fetch;
/** Skip files larger than this. Default 50 MiB. */
media-scrape/src/engine.ts
+20 -5
View File
@@ -4,7 +4,13 @@
*/
import { join } from "node:path";
import { downloadGapMs, downloadOne, downloadPreview, type DownloadOptions } from "./download.js";
import {
downloadGapMs,
downloadOne,
downloadPreview,
probeDimensions,
type DownloadOptions
} from "./download.js";
import {
createScrapeDirs,
listScrapes,
@@ -50,6 +56,8 @@ export interface ScrapeEngineOptions {
download?: DownloadOptions;
/** Cache Discord-hosted image previews after every walk. Default true. */
previews?: boolean;
/** Read dimensions from a downloaded file. Defaults to ffprobe. */
probe?: (path: string) => Promise<{ width: number; height: number } | null>;
}
interface QueuedJob {
@@ -244,6 +252,9 @@ export class ScrapeEngine {
meta.lastError = error instanceof Error ? error.message : String(error);
this.log(`${meta.id}: failed: ${meta.lastError}`);
} finally {
if ((meta.state === "done" || meta.state === "failed") && !meta.finishedAt) {
meta.finishedAt = new Date(this.now()).toISOString();
}
this.persist(dir, meta);
store.close();
this.active.set(meta.id, meta);
@@ -273,6 +284,11 @@ export class ScrapeEngine {
bytes: outcome.bytes ?? 0
});
meta.downloaded = (meta.downloaded ?? 0) + 1;
if (row.width === null || row.height === null) {
const dimensions = await (this.options.probe ?? probeDimensions)(join(dir, outcome.path));
if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height);
}
} else {
store.markDownloadError(row.id, outcome.error ?? outcome.state);
if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1;
@@ -283,11 +299,10 @@ export class ScrapeEngine {
}
}
/** Cache small previews. In download mode this includes embed thumbnails. */
/** Cache small previews. Embed thumbnails are cached in both modes. */
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const embedPreviews = meta.mode === "download";
const options = { ...(this.options.download ?? {}), embedPreviews };
const pending = store.pendingPreviews(5000, embedPreviews);
const options = { ...(this.options.download ?? {}), embedPreviews: true };
const pending = store.pendingPreviews(5000, true);
for (const row of pending) {
if (this.stopRequested.has(meta.id)) {
media-scrape/src/extract.ts
+2 -2
View File
@@ -76,8 +76,8 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
provider,
downloaded: false,
size: null,
width: null,
height: null,
width: embed.width ?? null,
height: embed.height ?? null,
durationSecs: null
};
media-scrape/src/filter-presets.ts
+50
View File
@@ -0,0 +1,50 @@
/*
* Query presets for the media filters: aspect ratio and resolution buckets.
* Pure; the SQL fragments are fixed strings, never built from user input.
*/
export const ASPECT_PRESETS = ["vertical", "square", "wide43", "wide169", "ultrawide", "other"] as const;
export type AspectPreset = (typeof ASPECT_PRESETS)[number];
/** Aspect-ratio buckets: vertical/shorts, square, 4:3, 16:9, 21:9, everything else. */
export function aspectCondition(preset: string): string | null {
switch (preset) {
case "vertical":
return "aspect_ratio < 0.95";
case "square":
return "aspect_ratio >= 0.95 AND aspect_ratio < 1.05";
case "wide43":
return "aspect_ratio >= 1.05 AND aspect_ratio < 1.55";
case "wide169":
return "aspect_ratio >= 1.55 AND aspect_ratio < 2.05";
case "ultrawide":
return "aspect_ratio >= 2.05 AND aspect_ratio < 2.8";
case "other":
return "(aspect_ratio IS NULL OR aspect_ratio >= 2.8)";
default:
return null;
}
}
export const RESOLUTION_PRESETS = ["sd", "hd", "fhd", "qhd", "uhd", "unknown"] as const;
export type ResolutionPreset = (typeof RESOLUTION_PRESETS)[number];
/** Resolution buckets by the shortest side, so vertical 1080x1920 counts as FHD. */
export function resolutionCondition(preset: string): string | null {
switch (preset) {
case "sd":
return "MIN(width, height) <= 480";
case "hd":
return "MIN(width, height) > 480 AND MIN(width, height) <= 900";
case "fhd":
return "MIN(width, height) > 900 AND MIN(width, height) <= 1260";
case "qhd":
return "MIN(width, height) > 1260 AND MIN(width, height) <= 1800";
case "uhd":
return "MIN(width, height) > 1800";
case "unknown":
return "(width IS NULL OR height IS NULL)";
default:
return null;
}
}
media-scrape/src/index.ts
+8
View File
@@ -5,6 +5,14 @@
export { extractMedia, type ExtractedMedia } from "./extract.js";
export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
export {
ASPECT_PRESETS,
RESOLUTION_PRESETS,
aspectCondition,
resolutionCondition,
type AspectPreset,
type ResolutionPreset
} from "./filter-presets.js";
export {
downloadGapMs,
downloadOne,
media-scrape/src/store.ts
+72 -2
View File
@@ -8,6 +8,7 @@ import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { fileURLToPath } from "node:url";
import { extractMedia } from "./extract.js";
import { aspectCondition, resolutionCondition } from "./filter-presets.js";
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
@@ -77,6 +78,12 @@ export function dirSizeBytes(dir: string): number {
return total;
}
/** Width / height, rounded to three decimals. Null when either side is missing or zero. */
export function aspectRatioOf(width: number | null | undefined, height: number | null | undefined): number | null {
if (!Number.isFinite(width) || !Number.isFinite(height) || !width || !height) return null;
return Math.round(((width as number) / (height as number)) * 1000) / 1000;
}
export function createScrapeDirs(root: string, id: string): string {
const dir = scrapeDir(root, id);
mkdirSync(join(dir, "media"), { recursive: true });
@@ -99,12 +106,16 @@ export interface MediaRow {
id: number;
kind: MediaKind;
source: string | null;
author_id: string | null;
author_name: string | null;
posted_at: string | null;
url: string;
provider: string | null;
content_type: string | null;
size: number | null;
width: number | null;
height: number | null;
aspect_ratio: number | null;
duration_secs: number | null;
downloaded: number;
preview_url: string | null;
@@ -131,12 +142,16 @@ CREATE TABLE IF NOT EXISTS media (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
source TEXT,
author_id TEXT,
author_name TEXT,
posted_at TEXT,
url TEXT NOT NULL UNIQUE,
provider TEXT,
content_type TEXT,
size INTEGER,
width INTEGER,
height INTEGER,
aspect_ratio REAL,
duration_secs REAL,
downloaded INTEGER NOT NULL DEFAULT 0,
preview_url TEXT,
@@ -157,6 +172,7 @@ CREATE TABLE IF NOT EXISTS media_sources (
);
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
CREATE INDEX IF NOT EXISTS idx_media_posted ON media(posted_at);
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
`;
@@ -178,9 +194,13 @@ export class ScrapeStore {
if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
if (!names.has("author_id")) this.db.exec("ALTER TABLE media ADD COLUMN author_id TEXT");
if (!names.has("author_name")) this.db.exec("ALTER TABLE media ADD COLUMN author_name TEXT");
if (!names.has("posted_at")) this.db.exec("ALTER TABLE media ADD COLUMN posted_at TEXT");
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
}
close(): void {
@@ -194,7 +214,7 @@ export class ScrapeStore {
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
);
const insertMedia = this.db.prepare(
"INSERT OR IGNORE INTO media (kind, source, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
"INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
);
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
const linkSource = this.db.prepare(
@@ -217,12 +237,16 @@ export class ScrapeStore {
const result = insertMedia.run(
item.kind,
item.source,
message.author.id,
message.author.username,
message.timestamp,
item.url,
item.provider,
item.contentType,
item.size,
item.width,
item.height,
aspectRatioOf(item.width, item.height),
item.durationSecs,
item.downloaded ? 1 : 0,
item.previewUrl,
@@ -321,6 +345,23 @@ export class ScrapeStore {
this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
}
/** Fill in dimensions after a download probe. */
updateDimensions(id: number, width: number, height: number): void {
this.db
.prepare("UPDATE media SET width = ?, height = ?, aspect_ratio = ? WHERE id = ?")
.run(width, height, aspectRatioOf(width, height), id);
}
/** Distinct posters with their item counts, biggest first. */
authors(limit = 200): Array<{ id: string | null; name: string | null; count: number }> {
const rows = this.db
.prepare(
"SELECT author_id AS id, author_name AS name, COUNT(*) AS count FROM media WHERE author_name IS NOT NULL GROUP BY author_id, author_name ORDER BY count DESC LIMIT ?"
)
.all(limit) as unknown as Array<{ id: string | null; name: string | null; count: number }>;
return rows;
}
getMedia(id: number): MediaRow | null {
const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
| MediaRow
@@ -329,13 +370,24 @@ export class ScrapeStore {
}
queryMedia(
options: { kind?: MediaKind; source?: string; limit?: number; offset?: number } = {}
options: {
kind?: MediaKind;
source?: string;
authorId?: string;
aspect?: string;
resolution?: string;
after?: string;
before?: string;
limit?: number;
offset?: number;
} = {}
): MediaRow[] {
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
const offset = Math.max(0, options.offset ?? 0);
const conditions: string[] = [];
const values: Array<string | number> = [];
if (options.kind) {
conditions.push("kind = ?");
values.push(options.kind);
@@ -344,6 +396,24 @@ export class ScrapeStore {
conditions.push("source = ?");
values.push(options.source);
}
if (options.authorId) {
conditions.push("author_id = ?");
values.push(options.authorId);
}
if (options.after) {
conditions.push("posted_at >= ?");
values.push(options.after);
}
if (options.before) {
conditions.push("posted_at <= ?");
values.push(options.before);
}
const aspect = options.aspect ? aspectCondition(options.aspect) : null;
if (aspect) conditions.push(aspect);
const resolution = options.resolution ? resolutionCondition(options.resolution) : null;
if (resolution) conditions.push(resolution);
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
const rows = this.db
media-scrape/src/types.ts
+5
View File
@@ -39,6 +39,9 @@ export interface CoreEmbed {
videoUrl: string | null;
imageUrl: string | null;
thumbnailUrl: string | null;
/** Dimensions of the video, image, or thumbnail, first one Discord provides. */
width?: number | null;
height?: number | null;
}
export interface CoreSticker {
@@ -89,6 +92,8 @@ export interface ScrapeMeta {
downloadErrors?: number;
/** Cached image previews. */
previewed?: number;
/** When the scrape reached done or failed. */
finishedAt?: string | null;
}
export function emptyMediaCounts(): Record<MediaKind, number> {
media-scrape/test/run.ts
+80 -5
View File
@@ -12,6 +12,7 @@ import { downloadOne, downloadPreview, previewUrlFor } from "../src/download.js"
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
import { extractMedia } from "../src/extract.js";
import { PuppetBackend } from "../src/puppet-backend.js";
import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
import { isNearLimit, planRetention } from "../src/retention.js";
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
import type { CoreAttachment, CoreMessage } from "../src/types.js";
@@ -196,6 +197,7 @@ void (async () => {
assert.equal(meta?.messages, 3);
assert.equal(meta?.cursor, "99");
assert.equal(meta?.media.image, 1);
assert.ok(meta?.finishedAt);
assert.equal(backend.calls[0].limit, 2);
assert.equal(backend.calls[1].before, "100");
@@ -209,6 +211,9 @@ void (async () => {
assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
assert.equal(stored.storedBytesByKind().image, 0);
const row = stored.queryMedia({ kind: "image" })[0];
assert.equal(row.author_name, "tester");
assert.equal(row.posted_at, "2026-09-16T00:00:00.000Z");
stored.close();
assert.equal(listScrapes(root).length, 1);
@@ -405,14 +410,30 @@ void (async () => {
assert.equal(existsSync(join(root, started.id, row.file_path)), true);
});
await test("store filters by source and kind", () => {
await test("aspect and resolution presets have fixed buckets", () => {
assert.equal(aspectCondition("vertical"), "aspect_ratio < 0.95");
assert.equal(aspectCondition("wide169"), "aspect_ratio >= 1.55 AND aspect_ratio < 2.05");
assert.equal(aspectCondition("nope"), null);
assert.equal(resolutionCondition("fhd"), "MIN(width, height) > 900 AND MIN(width, height) <= 1260");
assert.equal(resolutionCondition("unknown"), "(width IS NULL OR height IS NULL)");
assert.equal(resolutionCondition("nope"), null);
});
await test("store filters by source, kind, author, ratio, resolution, and dates", () => {
const dir = join(root, "store-source");
mkdirSync(dir, { recursive: true });
const store = new ScrapeStore(dir);
store.insertPage([
message("700", {
attachments: [attachment({ filename: "pic.png", contentType: "image/png" })],
attachments: [
attachment({
filename: "pic.png",
contentType: "image/png",
width: 640,
height: 360
})
],
embeds: [
{
type: "video",
@@ -432,6 +453,59 @@ void (async () => {
assert.equal(store.queryMedia({ source: "sticker" }).length, 0);
assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1);
assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0);
assert.equal(store.queryMedia({ aspect: "wide169" }).length, 1);
assert.equal(store.queryMedia({ aspect: "vertical" }).length, 0);
assert.equal(store.queryMedia({ resolution: "sd" }).length, 1);
assert.equal(store.queryMedia({ resolution: "fhd" }).length, 0);
assert.equal(store.queryMedia({ authorId: "1" }).length, 2);
assert.equal(store.queryMedia({ authorId: "999" }).length, 0);
assert.equal(store.queryMedia({ after: "2026-09-16T00:00:00.000Z" }).length, 2);
assert.equal(store.queryMedia({ before: "2026-09-15T00:00:00.000Z" }).length, 0);
const [wide] = store.queryMedia({ kind: "image" });
assert.equal(wide.aspect_ratio, 1.778);
store.close();
});
await test("engine probes dimensions when the metadata lacks them", async () => {
const backend = new FakeBackend([
{
messages: [
message("701", {
attachments: [attachment({ filename: "probe.png", contentType: "image/png" })]
})
]
}
]);
const engine = new ScrapeEngine({
rootDir: root,
pageSize: 2,
pageMinMs: 0,
pageMaxMs: 0,
sleep: async () => {},
random: () => 0,
log: () => {},
previews: false,
probe: async () => ({ width: 1080, height: 1920 }),
download: {
fetchImpl: async () =>
new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
minGapMs: 0,
maxGapMs: 0,
sleep: async () => {}
}
});
const started = await engine.start({ channelId: "129", guildId: null, mode: "download", backend });
await engine.processQueue();
const store = new ScrapeStore(join(root, started.id));
const row = store.queryMedia({ kind: "image" })[0];
assert.equal(row.width, 1080);
assert.equal(row.height, 1920);
assert.equal(row.aspect_ratio, 0.563);
store.close();
});
@@ -502,7 +576,7 @@ void (async () => {
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
});
await test("engine caches previews in links mode and skips embeds", async () => {
await test("engine caches previews in links mode, including embed thumbnails", async () => {
const backend = new FakeBackend([
{
messages: [
@@ -551,8 +625,9 @@ void (async () => {
const meta = engine.get(started.id);
assert.equal(meta?.state, "done");
assert.equal(meta?.previewed, 1);
assert.equal(fetched.some(url => url.includes("ytimg.com")), false);
assert.equal(meta?.previewed, 2);
assert.equal(fetched.some(url => url.includes("ytimg.com")), true);
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
const store = new ScrapeStore(join(root, started.id));
const row = store.queryMedia({ kind: "image" })[0];