zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add links-mode embed thumbnails, item metadata, and media filters

Browse Source
This commit is contained in:
Fisto
2026-09-16 11:44:09 +02:00
parent 1fe204b91e
commit 973b9cff92
23 changed files with 863 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/download.ts
+25
View File
@@ -3,11 +3,36 @@
* Rules: docs/media-scraper.md. Third-party embeds are never targets here.
*/
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import { promisify } from "node:util";
import type { MediaRow } from "./store.js";
const execFileAsync = promisify(execFile);
/** Read pixel dimensions with ffprobe. Null when the tool is missing or the file is not media. */
export async function probeDimensions(
path: string,
ffprobe = process.env.VESKSLOP_FFPROBE ?? "ffprobe"
): Promise<{ width: number; height: number } | null> {
try {
const { stdout } = await execFileAsync(
ffprobe,
["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "json", path],
{ timeout: 15_000 }
);
const stream = JSON.parse(stdout)?.streams?.[0];
if (Number.isFinite(stream?.width) && Number.isFinite(stream?.height)) {
return { width: stream.width, height: stream.height };
}
} catch {
// missing ffprobe, a timeout, or a non-visual file
}
return null;
}
export interface DownloadOptions {
fetchImpl?: typeof fetch;
/** Skip files larger than this. Default 50 MiB. */
media-scrape/src/engine.ts
+20 -5
View File
@@ -4,7 +4,13 @@
*/
import { join } from "node:path";
import { downloadGapMs, downloadOne, downloadPreview, type DownloadOptions } from "./download.js";
import {
downloadGapMs,
downloadOne,
downloadPreview,
probeDimensions,
type DownloadOptions
} from "./download.js";
import {
createScrapeDirs,
listScrapes,
@@ -50,6 +56,8 @@ export interface ScrapeEngineOptions {
download?: DownloadOptions;
/** Cache Discord-hosted image previews after every walk. Default true. */
previews?: boolean;
/** Read dimensions from a downloaded file. Defaults to ffprobe. */
probe?: (path: string) => Promise<{ width: number; height: number } | null>;
}
interface QueuedJob {
@@ -244,6 +252,9 @@ export class ScrapeEngine {
meta.lastError = error instanceof Error ? error.message : String(error);
this.log(`${meta.id}: failed: ${meta.lastError}`);
} finally {
if ((meta.state === "done" || meta.state === "failed") && !meta.finishedAt) {
meta.finishedAt = new Date(this.now()).toISOString();
}
this.persist(dir, meta);
store.close();
this.active.set(meta.id, meta);
@@ -273,6 +284,11 @@ export class ScrapeEngine {
bytes: outcome.bytes ?? 0
});
meta.downloaded = (meta.downloaded ?? 0) + 1;
if (row.width === null || row.height === null) {
const dimensions = await (this.options.probe ?? probeDimensions)(join(dir, outcome.path));
if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height);
}
} else {
store.markDownloadError(row.id, outcome.error ?? outcome.state);
if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1;
@@ -283,11 +299,10 @@ export class ScrapeEngine {
}
}
/** Cache small previews. In download mode this includes embed thumbnails. */
/** Cache small previews. Embed thumbnails are cached in both modes. */
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const embedPreviews = meta.mode === "download";
const options = { ...(this.options.download ?? {}), embedPreviews };
const pending = store.pendingPreviews(5000, embedPreviews);
const options = { ...(this.options.download ?? {}), embedPreviews: true };
const pending = store.pendingPreviews(5000, true);
for (const row of pending) {
if (this.stopRequested.has(meta.id)) {
media-scrape/src/extract.ts
+2 -2
View File
@@ -76,8 +76,8 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
provider,
downloaded: false,
size: null,
width: null,
height: null,
width: embed.width ?? null,
height: embed.height ?? null,
durationSecs: null
};
media-scrape/src/filter-presets.ts
+50
View File
@@ -0,0 +1,50 @@
/*
* Query presets for the media filters: aspect ratio and resolution buckets.
* Pure; the SQL fragments are fixed strings, never built from user input.
*/
export const ASPECT_PRESETS = ["vertical", "square", "wide43", "wide169", "ultrawide", "other"] as const;
export type AspectPreset = (typeof ASPECT_PRESETS)[number];
/** Aspect-ratio buckets: vertical/shorts, square, 4:3, 16:9, 21:9, everything else. */
export function aspectCondition(preset: string): string | null {
switch (preset) {
case "vertical":
return "aspect_ratio < 0.95";
case "square":
return "aspect_ratio >= 0.95 AND aspect_ratio < 1.05";
case "wide43":
return "aspect_ratio >= 1.05 AND aspect_ratio < 1.55";
case "wide169":
return "aspect_ratio >= 1.55 AND aspect_ratio < 2.05";
case "ultrawide":
return "aspect_ratio >= 2.05 AND aspect_ratio < 2.8";
case "other":
return "(aspect_ratio IS NULL OR aspect_ratio >= 2.8)";
default:
return null;
}
}
export const RESOLUTION_PRESETS = ["sd", "hd", "fhd", "qhd", "uhd", "unknown"] as const;
export type ResolutionPreset = (typeof RESOLUTION_PRESETS)[number];
/** Resolution buckets by the shortest side, so vertical 1080x1920 counts as FHD. */
export function resolutionCondition(preset: string): string | null {
switch (preset) {
case "sd":
return "MIN(width, height) <= 480";
case "hd":
return "MIN(width, height) > 480 AND MIN(width, height) <= 900";
case "fhd":
return "MIN(width, height) > 900 AND MIN(width, height) <= 1260";
case "qhd":
return "MIN(width, height) > 1260 AND MIN(width, height) <= 1800";
case "uhd":
return "MIN(width, height) > 1800";
case "unknown":
return "(width IS NULL OR height IS NULL)";
default:
return null;
}
}
media-scrape/src/index.ts
+8
View File
@@ -5,6 +5,14 @@
export { extractMedia, type ExtractedMedia } from "./extract.js";
export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
export {
ASPECT_PRESETS,
RESOLUTION_PRESETS,
aspectCondition,
resolutionCondition,
type AspectPreset,
type ResolutionPreset
} from "./filter-presets.js";
export {
downloadGapMs,
downloadOne,
media-scrape/src/store.ts
+72 -2
View File
@@ -8,6 +8,7 @@ import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { fileURLToPath } from "node:url";
import { extractMedia } from "./extract.js";
import { aspectCondition, resolutionCondition } from "./filter-presets.js";
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
@@ -77,6 +78,12 @@ export function dirSizeBytes(dir: string): number {
return total;
}
/** Width / height, rounded to three decimals. Null when either side is missing or zero. */
export function aspectRatioOf(width: number | null | undefined, height: number | null | undefined): number | null {
if (!Number.isFinite(width) || !Number.isFinite(height) || !width || !height) return null;
return Math.round(((width as number) / (height as number)) * 1000) / 1000;
}
export function createScrapeDirs(root: string, id: string): string {
const dir = scrapeDir(root, id);
mkdirSync(join(dir, "media"), { recursive: true });
@@ -99,12 +106,16 @@ export interface MediaRow {
id: number;
kind: MediaKind;
source: string | null;
author_id: string | null;
author_name: string | null;
posted_at: string | null;
url: string;
provider: string | null;
content_type: string | null;
size: number | null;
width: number | null;
height: number | null;
aspect_ratio: number | null;
duration_secs: number | null;
downloaded: number;
preview_url: string | null;
@@ -131,12 +142,16 @@ CREATE TABLE IF NOT EXISTS media (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
source TEXT,
author_id TEXT,
author_name TEXT,
posted_at TEXT,
url TEXT NOT NULL UNIQUE,
provider TEXT,
content_type TEXT,
size INTEGER,
width INTEGER,
height INTEGER,
aspect_ratio REAL,
duration_secs REAL,
downloaded INTEGER NOT NULL DEFAULT 0,
preview_url TEXT,
@@ -157,6 +172,7 @@ CREATE TABLE IF NOT EXISTS media_sources (
);
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
CREATE INDEX IF NOT EXISTS idx_media_posted ON media(posted_at);
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
`;
@@ -178,9 +194,13 @@ export class ScrapeStore {
if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
if (!names.has("author_id")) this.db.exec("ALTER TABLE media ADD COLUMN author_id TEXT");
if (!names.has("author_name")) this.db.exec("ALTER TABLE media ADD COLUMN author_name TEXT");
if (!names.has("posted_at")) this.db.exec("ALTER TABLE media ADD COLUMN posted_at TEXT");
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
}
close(): void {
@@ -194,7 +214,7 @@ export class ScrapeStore {
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
);
const insertMedia = this.db.prepare(
"INSERT OR IGNORE INTO media (kind, source, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
"INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
);
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
const linkSource = this.db.prepare(
@@ -217,12 +237,16 @@ export class ScrapeStore {
const result = insertMedia.run(
item.kind,
item.source,
message.author.id,
message.author.username,
message.timestamp,
item.url,
item.provider,
item.contentType,
item.size,
item.width,
item.height,
aspectRatioOf(item.width, item.height),
item.durationSecs,
item.downloaded ? 1 : 0,
item.previewUrl,
@@ -321,6 +345,23 @@ export class ScrapeStore {
this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
}
/** Fill in dimensions after a download probe. */
updateDimensions(id: number, width: number, height: number): void {
this.db
.prepare("UPDATE media SET width = ?, height = ?, aspect_ratio = ? WHERE id = ?")
.run(width, height, aspectRatioOf(width, height), id);
}
/** Distinct posters with their item counts, biggest first. */
authors(limit = 200): Array<{ id: string | null; name: string | null; count: number }> {
const rows = this.db
.prepare(
"SELECT author_id AS id, author_name AS name, COUNT(*) AS count FROM media WHERE author_name IS NOT NULL GROUP BY author_id, author_name ORDER BY count DESC LIMIT ?"
)
.all(limit) as unknown as Array<{ id: string | null; name: string | null; count: number }>;
return rows;
}
getMedia(id: number): MediaRow | null {
const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
| MediaRow
@@ -329,13 +370,24 @@ export class ScrapeStore {
}
queryMedia(
options: { kind?: MediaKind; source?: string; limit?: number; offset?: number } = {}
options: {
kind?: MediaKind;
source?: string;
authorId?: string;
aspect?: string;
resolution?: string;
after?: string;
before?: string;
limit?: number;
offset?: number;
} = {}
): MediaRow[] {
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
const offset = Math.max(0, options.offset ?? 0);
const conditions: string[] = [];
const values: Array<string | number> = [];
if (options.kind) {
conditions.push("kind = ?");
values.push(options.kind);
@@ -344,6 +396,24 @@ export class ScrapeStore {
conditions.push("source = ?");
values.push(options.source);
}
if (options.authorId) {
conditions.push("author_id = ?");
values.push(options.authorId);
}
if (options.after) {
conditions.push("posted_at >= ?");
values.push(options.after);
}
if (options.before) {
conditions.push("posted_at <= ?");
values.push(options.before);
}
const aspect = options.aspect ? aspectCondition(options.aspect) : null;
if (aspect) conditions.push(aspect);
const resolution = options.resolution ? resolutionCondition(options.resolution) : null;
if (resolution) conditions.push(resolution);
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
const rows = this.db
media-scrape/src/types.ts
+5
View File
@@ -39,6 +39,9 @@ export interface CoreEmbed {
videoUrl: string | null;
imageUrl: string | null;
thumbnailUrl: string | null;
/** Dimensions of the video, image, or thumbnail, first one Discord provides. */
width?: number | null;
height?: number | null;
}
export interface CoreSticker {
@@ -89,6 +92,8 @@ export interface ScrapeMeta {
downloadErrors?: number;
/** Cached image previews. */
previewed?: number;
/** When the scrape reached done or failed. */
finishedAt?: string | null;
}
export function emptyMediaCounts(): Record<MediaKind, number> {