zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add links-mode embed thumbnails, item metadata, and media filters

Browse Source
This commit is contained in:
Fisto
2026-09-16 11:44:09 +02:00
parent 1fe204b91e
commit 973b9cff92
23 changed files with 863 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/store.ts
+72 -2
View File
@@ -8,6 +8,7 @@ import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { fileURLToPath } from "node:url";
import { extractMedia } from "./extract.js";
import { aspectCondition, resolutionCondition } from "./filter-presets.js";
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
@@ -77,6 +78,12 @@ export function dirSizeBytes(dir: string): number {
return total;
}
/** Width / height, rounded to three decimals. Null when either side is missing or zero. */
export function aspectRatioOf(width: number | null | undefined, height: number | null | undefined): number | null {
if (!Number.isFinite(width) || !Number.isFinite(height) || !width || !height) return null;
return Math.round(((width as number) / (height as number)) * 1000) / 1000;
}
export function createScrapeDirs(root: string, id: string): string {
const dir = scrapeDir(root, id);
mkdirSync(join(dir, "media"), { recursive: true });
@@ -99,12 +106,16 @@ export interface MediaRow {
id: number;
kind: MediaKind;
source: string | null;
author_id: string | null;
author_name: string | null;
posted_at: string | null;
url: string;
provider: string | null;
content_type: string | null;
size: number | null;
width: number | null;
height: number | null;
aspect_ratio: number | null;
duration_secs: number | null;
downloaded: number;
preview_url: string | null;
@@ -131,12 +142,16 @@ CREATE TABLE IF NOT EXISTS media (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
source TEXT,
author_id TEXT,
author_name TEXT,
posted_at TEXT,
url TEXT NOT NULL UNIQUE,
provider TEXT,
content_type TEXT,
size INTEGER,
width INTEGER,
height INTEGER,
aspect_ratio REAL,
duration_secs REAL,
downloaded INTEGER NOT NULL DEFAULT 0,
preview_url TEXT,
@@ -157,6 +172,7 @@ CREATE TABLE IF NOT EXISTS media_sources (
);
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
CREATE INDEX IF NOT EXISTS idx_media_posted ON media(posted_at);
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
`;
@@ -178,9 +194,13 @@ export class ScrapeStore {
if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
if (!names.has("author_id")) this.db.exec("ALTER TABLE media ADD COLUMN author_id TEXT");
if (!names.has("author_name")) this.db.exec("ALTER TABLE media ADD COLUMN author_name TEXT");
if (!names.has("posted_at")) this.db.exec("ALTER TABLE media ADD COLUMN posted_at TEXT");
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
}
close(): void {
@@ -194,7 +214,7 @@ export class ScrapeStore {
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
);
const insertMedia = this.db.prepare(
"INSERT OR IGNORE INTO media (kind, source, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
"INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
);
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
const linkSource = this.db.prepare(
@@ -217,12 +237,16 @@ export class ScrapeStore {
const result = insertMedia.run(
item.kind,
item.source,
message.author.id,
message.author.username,
message.timestamp,
item.url,
item.provider,
item.contentType,
item.size,
item.width,
item.height,
aspectRatioOf(item.width, item.height),
item.durationSecs,
item.downloaded ? 1 : 0,
item.previewUrl,
@@ -321,6 +345,23 @@ export class ScrapeStore {
this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
}
/** Fill in dimensions after a download probe. */
updateDimensions(id: number, width: number, height: number): void {
this.db
.prepare("UPDATE media SET width = ?, height = ?, aspect_ratio = ? WHERE id = ?")
.run(width, height, aspectRatioOf(width, height), id);
}
/** Distinct posters with their item counts, biggest first. */
authors(limit = 200): Array<{ id: string | null; name: string | null; count: number }> {
const rows = this.db
.prepare(
"SELECT author_id AS id, author_name AS name, COUNT(*) AS count FROM media WHERE author_name IS NOT NULL GROUP BY author_id, author_name ORDER BY count DESC LIMIT ?"
)
.all(limit) as unknown as Array<{ id: string | null; name: string | null; count: number }>;
return rows;
}
getMedia(id: number): MediaRow | null {
const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
| MediaRow
@@ -329,13 +370,24 @@ export class ScrapeStore {
}
queryMedia(
options: { kind?: MediaKind; source?: string; limit?: number; offset?: number } = {}
options: {
kind?: MediaKind;
source?: string;
authorId?: string;
aspect?: string;
resolution?: string;
after?: string;
before?: string;
limit?: number;
offset?: number;
} = {}
): MediaRow[] {
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
const offset = Math.max(0, options.offset ?? 0);
const conditions: string[] = [];
const values: Array<string | number> = [];
if (options.kind) {
conditions.push("kind = ?");
values.push(options.kind);
@@ -344,6 +396,24 @@ export class ScrapeStore {
conditions.push("source = ?");
values.push(options.source);
}
if (options.authorId) {
conditions.push("author_id = ?");
values.push(options.authorId);
}
if (options.after) {
conditions.push("posted_at >= ?");
values.push(options.after);
}
if (options.before) {
conditions.push("posted_at <= ?");
values.push(options.before);
}
const aspect = options.aspect ? aspectCondition(options.aspect) : null;
if (aspect) conditions.push(aspect);
const resolution = options.resolution ? resolutionCondition(options.resolution) : null;
if (resolution) conditions.push(resolution);
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
const rows = this.db