Add links-mode embed thumbnails, item metadata, and media filters

This commit is contained in:
Fisto
2026-09-16 11:44:09 +02:00
parent 1fe204b91e
commit 973b9cff92
23 changed files with 863 additions and 82 deletions
+9
View File
@@ -40,6 +40,15 @@ mobile draft at 390px.
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200). `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
- [x] C14 Links-mode embed thumbnails, item metadata, filters, and the scrape popup. Metadata:
poster (`author_id`/`author_name`), `posted_at`, `width`/`height`, `aspect_ratio`, with an
`ffprobe` fallback after downloads. Filters: poster, aspect preset (vertical/square/4:3/16:9/
21:9/other), resolution preset (SD/HD/FHD/QHD/4K+/unknown, by shortest side), and a
posted-date range. UI: scrape popup with Server and Group-chat trees (server -> channels, each
with Links/Download), History popup (time, duration, pages/messages, media, bytes, files,
previews), live progress pulse, full-window view. Live run `...-20260916094120`: 11 previews
in links mode (embed thumbnails included), poster zzib, a 9:16 short stored as 0.563, and the
16:9 filter matched.
- [x] C13 Embed thumbnails in download mode: embed rows cache their stored thumbnail URL; - [x] C13 Embed thumbnails in download mode: embed rows cache their stored thumbnail URL;
the full third-party media is never fetched. Live run `...-20260916091458` on the test the full third-party media is never fetched. Live run `...-20260916091458` on the test
channel: 6 files downloaded, 11 previews cached (YouTube, Reddit, Tenor, TikTok, X). Also channel: 6 files downloaded, 11 previews cached (YouTube, Reddit, Tenor, TikTok, X). Also
+1 -1
View File
@@ -67,7 +67,7 @@ Event line:
| `user.search` | `query`, optional `guildId`, optional `limit` | Users from the client cache. Guild members when `guildId` is set. | | `user.search` | `query`, optional `guildId`, optional `limit` | Users from the client cache. Guild members when `guildId` is set. |
| `dm.list` | optional `limit` | Cached DM and group DM channels with recipients, icons, and avatars. Cache-only. | | `dm.list` | optional `limit` | Cached DM and group DM channels with recipients, icons, and avatars. Cache-only. |
| `message.send` | `channelId`, `content` | `{ channelId, messageId, sent: true }` | | `message.send` | `channelId`, `content` | `{ channelId, messageId, sent: true }` |
| `message.read` | `channelId`, `limit` | `{ cached, messages }`. Cache-only. Messages carry the author avatar, attachment metadata (id, filename, URL, content type, size, dimensions, duration), embeds (video, image, and thumbnail URLs), sticker items, forwarded snapshots, a voice-message flag, `editedAt`, and `replyTo`. | | `message.read` | `channelId`, `limit` | `{ cached, messages }`. Cache-only. Messages carry the author avatar, attachment metadata (id, filename, URL, content type, size, dimensions, duration), embeds (video, image, and thumbnail URLs plus their dimensions), sticker items, forwarded snapshots, a voice-message flag, `editedAt`, and `replyTo`. |
| `message.fetch` | `channelId`, `limit`, optional `before` or `after` | `{ fetched, messages }`. Fetches from Discord. Same message shape. | | `message.fetch` | `channelId`, `limit`, optional `before` or `after` | `{ fetched, messages }`. Fetches from Discord. Same message shape. |
| `reaction.add` | `channelId`, `messageId`, `emoji` | `{ added: true }` | | `reaction.add` | `channelId`, `messageId`, `emoji` | `{ added: true }` |
| `dm.send` | `userId`, `content` | `{ userId, channelId, sent: true }` | | `dm.send` | `userId`, `content` | `{ userId, channelId, sent: true }` |
+1
View File
@@ -175,6 +175,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the
| `VESKSLOP_MEDIA_DB_DIR` | `<repo>/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. | | `VESKSLOP_MEDIA_DB_DIR` | `<repo>/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. |
| `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. | | `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. |
| `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. | | `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. |
| `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. |
| `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. | | `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. |
| `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. | | `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. |
| `VESKSLOP_TTS_LEAD_MS` | `150` | Lead-in silence before each TTS line (0 disables, max 2000). | | `VESKSLOP_TTS_LEAD_MS` | `150` | Lead-in silence before each TTS line (0 disables, max 2000). |
+4 -2
View File
@@ -23,8 +23,10 @@ interface MediaService {
} }
``` ```
`MediaQuery` (implemented): `kind`, `source`, `scrapeId`, `channelId`, `limit`, `offset`. Planned: `MediaQuery` (implemented): `kind`, `source`, `scrapeId`, `channelId`, `authorId`, `aspect`,
`guildId`, `authorId`, `after`, `before`, `text`. `resolution`, `after`, `before`, `limit`, `offset`. The `aspect` and `resolution` values are the
preset keys from `filter-presets.ts`; `after`/`before` are ISO times on `posted_at`. Planned:
`guildId`, `text`.
`StorageStatus`: `usedBytes`, `limitBytes`, `warnPercent`, `overrunPercent`, `byKind`, `StorageStatus`: `usedBytes`, `limitBytes`, `warnPercent`, `overrunPercent`, `byKind`,
`scrapes: [{ id, bytes, state, createdAt }]`. `scrapes: [{ id, bytes, state, createdAt }]`.
+10 -6
View File
@@ -81,12 +81,16 @@ One SQLite file per scrape (`index.db`, `node:sqlite`).
| Table | Columns | | Table | Columns |
| --- | --- | | --- | --- |
| `messages` | `id`, `author_id`, `author_name`, `timestamp`, `content` | | `messages` | `id`, `author_id`, `author_name`, `timestamp`, `content` |
| `media` | `id`, `kind`, `url`, `provider`, `file_hash`, `preview_hash`, `content_type`, `size`, `width`, `height`, `duration_ms`, `downloaded`, `created_at` | | `media` | `id`, `kind`, `source`, `author_id`, `author_name`, `posted_at`, `url`, `provider`, `file_hash`, `preview_hash`, `content_type`, `size`, `width`, `height`, `aspect_ratio`, `duration_secs`, `downloaded`, `file_path`, `created_at` |
| `media_sources` | `media_id`, `message_id` | | `media_sources` | `media_id`, `message_id` |
| `downloads` | `media_id`, `state`, `attempts`, `error` | | `downloads` | `media_id`, `state`, `attempts`, `error` |
Indexes: `(kind, timestamp)` via `media_sources`, `channel` metadata in `scrape.json`, and Indexes: `kind`, `posted_at`, and `media_sources(message_id)`. The database is an index; the media
`file_hash` for dedupe. The database is an index; the media folder holds bytes. folder holds bytes. Every item carries its poster (`author_id`, `author_name`), the post time
(`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). Filter presets live in
`media-scrape/src/filter-presets.ts`: aspect buckets (vertical, square, 4:3, 16:9, 21:9, other) and
resolution buckets by the shortest side (SD, HD, FHD, QHD, 4K+, unknown). A missing resolution is
filled with `ffprobe` after a download when `VESKSLOP_FFPROBE` is available.
## Downloads and previews ## Downloads and previews
@@ -105,9 +109,9 @@ Previews run after every walk:
- In both modes, Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`) - In both modes, Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`)
are transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`). are transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`).
- In `download` mode only, embed rows also get their stored `preview_url` (YouTube, X, TikTok - In both modes, embed rows also get their stored `preview_url` (YouTube, X, TikTok thumbnails,
thumbnails, and so on) cached. The embed's full media URL is never fetched; embed rows are and so on) cached. The embed's full media URL is never fetched; embed rows are excluded from
excluded from `pendingDownloads`. `pendingDownloads`.
- One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape. - One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape.
- Stored by hash in `thumbs/<sha256>.webp`; `preview_hash` and `preview_bytes` land on the row, and - Stored by hash in `thumbs/<sha256>.webp`; `preview_hash` and `preview_bytes` land on the row, and
the counter is `previewed`. Previews count toward the storage limit. the counter is `previewed`. Previews count toward the storage limit.
+16 -10
View File
@@ -81,11 +81,13 @@ media-scrape/db/
are never download targets. The Media tab prefers the local file when one exists, so the archive are never download targets. The Media tab prefers the local file when one exists, so the archive
survives CDN URL expiry. survives CDN URL expiry.
- Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every - Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every
other embed stays a link. In `download` mode their thumbnail is cached (below); the video itself other embed stays a link. Its thumbnail is cached (below); the video itself is never fetched.
is never fetched. - Previews are cached small (WebP, 320px) in `thumbs/` after every walk, in both modes: Discord
- Previews are cached small (WebP, 320px) in `thumbs/` after every walk. In both modes this covers hosted images and GIFs, and third-party embed thumbnails (YouTube, X, TikTok, and so on), so the
Discord-hosted images and GIFs; in `download` mode it also covers embed thumbnails, so the grid grid does not depend on expiring CDN links. Previews count toward the storage limit.
does not depend on expiring CDN links. Previews count toward the storage limit. - Every item records who posted it, when, and its dimensions: `author_id`, `author_name`,
`posted_at`, `width`, `height`, and `aspect_ratio`. In `download` mode a missing resolution is
filled in with `ffprobe` after the file lands.
- Discord attachment URLs are signed and expire in about a day. In `links` mode an expired URL is - Discord attachment URLs are signed and expire in about a day. In `links` mode an expired URL is
refreshed by re-fetching its message; in `download` mode the bytes are caught before expiry. refreshed by re-fetching its message; in `download` mode the bytes are caught before expiry.
@@ -127,11 +129,15 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun
## Web UI ## Web UI
- **Media** tab: filter chips (All, Images, GIFs, Videos, Audio), a source filter (any, - **Media** tab: filter chips (All, Images, GIFs, Videos, Audio), and filters for source (any,
attachments, stickers, embeds), and a source-channel filter built from the scrape list. Date and attachments, stickers, embeds), source channel, poster, aspect ratio (Vertical, 1:1, 4:3, 16:9,
text search are planned. Grid with lazy loading; previews come from the cache, then the local 21:9, Other), resolution (SD, HD, FHD, QHD, 4K+, Unknown), and a posted-date range (from/to).
file, then the CDN. A scrape-status strip shows progress with Start and Stop. The view fills the Grid with lazy loading; previews come from the cache, then the local file, then the CDN; each
window below the header and the call strip. tile's tooltip shows the poster and posted time. A scrape strip shows live progress with Stop.
**New scrape…** opens a popup with Server and Group chat trees (servers expand to channels; each
row has Links and Download), and **History** lists each scrape with its start time, duration,
pages, messages, media counts, bytes, files, and previews. The view fills the window below the
header and the call strip.
- **Bot** tab: slopbot connection state, the scrape queue, and the storage limit with a usage bar. - **Bot** tab: slopbot connection state, the scrape queue, and the storage limit with a usage bar.
The bar is segmented and colour coded by media kind (images, GIFs, video, audio, other) plus the The bar is segmented and colour coded by media kind (images, GIFs, video, audio, other) plus the
database overhead. database overhead.
+11 -6
View File
@@ -124,11 +124,15 @@ box live in Chat, not Call, and the connection stats are not chips yet.
- **Media** and **Bot** are scope buttons beside Guild and DM. They open a full-window view below - **Media** and **Bot** are scope buttons beside Guild and DM. They open a full-window view below
the header and the call strip (the shell is covered; only the header, and the call strip when it the header and the call strip (the shell is covered; only the header, and the call strip when it
is visible, stay on screen). Mobile is out of scope for the media feature. is visible, stay on screen). Mobile is out of scope for the media feature.
- Media lists scraped items with kind chips (All, Images, GIFs, Videos, Audio), a source filter - Media lists scraped items with kind chips (All, Images, GIFs, Videos, Audio) and filters for
(any, attachments, stickers, embeds), a source-channel filter built from the scrape list, a source (any, attachments, stickers, embeds), source channel, poster, aspect ratio, resolution,
channel picker with **Start links** / **Start download**, and the active scrapes with **Stop**. and a posted-date range (from/to). **New scrape…** opens a popup with **Server** and **Group
Images and GIFs render from the cached preview when one exists, then the local file, then the chat** tabs: servers expand into their text channels, each channel (and each group chat) has
Discord CDN URL; videos and audio are link cards. **Links** and **Download** buttons, and group chats always use the puppet. **History** lists
each scrape with start time, duration, pages, messages, media counts, bytes, files, and previews.
The scrape strip shows live progress with a pulse and **Stop**. Images and GIFs render from the
cached preview when one exists, then the local file, then the CDN URL; videos and audio are link
cards with the poster and posted time in the tooltip.
- Bot shows the slopbot connection state and the scrape queue. - Bot shows the slopbot connection state and the scrape queue.
- The views are self-contained: `poc/public/media.js` and `media.css`, over the `/api/media/*` - The views are self-contained: `poc/public/media.js` and `media.css`, over the `/api/media/*`
routes in the table below. See [media.md](media.md). routes in the table below. See [media.md](media.md).
@@ -252,11 +256,12 @@ replaceable.
| POST | `/api/audio/stop` | none | Stop speech and clear the queue | | POST | `/api/audio/stop` | none | Stop speech and clear the queue |
| POST | `/api/audio/capture` | `start` | Start or stop capture | | POST | `/api/audio/capture` | `start` | Start or stop capture |
| GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments | | GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments |
| GET | `/api/media` | `kind?`, `source?`, `channelId?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `channelId` filters by source channel. `localUrl`/`thumbUrl` when a file or preview exists | | GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?`, `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`; `localUrl`/`thumbUrl` when a file or preview exists |
| GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension | | GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension |
| GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached WebP preview | | GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached WebP preview |
| GET | `/api/media/scrapes` | none | Every scrape with state and counters | | GET | `/api/media/scrapes` | none | Every scrape with state and counters |
| GET | `/api/media/status` | none | Slopbot connection state and the scrape list | | GET | `/api/media/status` | none | Slopbot connection state and the scrape list |
| GET | `/api/media/authors` | none | Distinct posters across every scrape, with counts |
| GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over | | GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over |
| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `backend?`, `confirm?` | Start a media scrape. `backend` is `auto` (default), `slopbot`, or `puppet`. Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` | | POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `backend?`, `confirm?` | Start a media scrape. `backend` is `auto` (default), `slopbot`, or `puppet`. Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` |
| POST | `/api/media/scrape/stop` | `id` | Stop a media scrape | | POST | `/api/media/scrape/stop` | `id` | Stop a media scrape |
+14 -1
View File
@@ -37,11 +37,22 @@ function mapAttachments(message: any): CoreAttachment[] {
})); }));
} }
/** Dimensions of the first embed part that has them: video, image, then thumbnail. */
function embedDataSize(data: any) {
for (const part of [data?.video, data?.image, data?.thumbnail]) {
if (part && Number.isFinite(part.width) && Number.isFinite(part.height)) {
return { width: part.width, height: part.height };
}
}
return { width: null, height: null };
}
function mapEmbeds(message: any): CoreEmbed[] { function mapEmbeds(message: any): CoreEmbed[] {
const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null); const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null);
return collectionValues(message?.embeds).map((embed: any) => { return collectionValues(message?.embeds).map((embed: any) => {
const data = embed?.data ?? embed ?? {}; const data = embed?.data ?? embed ?? {};
const size = embedDataSize(data);
return { return {
type: data.type ?? null, type: data.type ?? null,
url: data.url ?? null, url: data.url ?? null,
@@ -49,7 +60,9 @@ function mapEmbeds(message: any): CoreEmbed[] {
provider: data.provider?.name ?? null, provider: data.provider?.name ?? null,
videoUrl: assetUrl(data.video), videoUrl: assetUrl(data.video),
imageUrl: assetUrl(data.image), imageUrl: assetUrl(data.image),
thumbnailUrl: assetUrl(data.thumbnail) thumbnailUrl: assetUrl(data.thumbnail),
width: size.width,
height: size.height
}; };
}); });
} }
+25
View File
@@ -3,11 +3,36 @@
* Rules: docs/media-scraper.md. Third-party embeds are never targets here. * Rules: docs/media-scraper.md. Third-party embeds are never targets here.
*/ */
import { execFile } from "node:child_process";
import { createHash } from "node:crypto"; import { createHash } from "node:crypto";
import { writeFileSync } from "node:fs"; import { writeFileSync } from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
import { promisify } from "node:util";
import type { MediaRow } from "./store.js"; import type { MediaRow } from "./store.js";
const execFileAsync = promisify(execFile);
/** Read pixel dimensions with ffprobe. Null when the tool is missing or the file is not media. */
export async function probeDimensions(
path: string,
ffprobe = process.env.VESKSLOP_FFPROBE ?? "ffprobe"
): Promise<{ width: number; height: number } | null> {
try {
const { stdout } = await execFileAsync(
ffprobe,
["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "json", path],
{ timeout: 15_000 }
);
const stream = JSON.parse(stdout)?.streams?.[0];
if (Number.isFinite(stream?.width) && Number.isFinite(stream?.height)) {
return { width: stream.width, height: stream.height };
}
} catch {
// missing ffprobe, a timeout, or a non-visual file
}
return null;
}
export interface DownloadOptions { export interface DownloadOptions {
fetchImpl?: typeof fetch; fetchImpl?: typeof fetch;
/** Skip files larger than this. Default 50 MiB. */ /** Skip files larger than this. Default 50 MiB. */
+20 -5
View File
@@ -4,7 +4,13 @@
*/ */
import { join } from "node:path"; import { join } from "node:path";
import { downloadGapMs, downloadOne, downloadPreview, type DownloadOptions } from "./download.js"; import {
downloadGapMs,
downloadOne,
downloadPreview,
probeDimensions,
type DownloadOptions
} from "./download.js";
import { import {
createScrapeDirs, createScrapeDirs,
listScrapes, listScrapes,
@@ -50,6 +56,8 @@ export interface ScrapeEngineOptions {
download?: DownloadOptions; download?: DownloadOptions;
/** Cache Discord-hosted image previews after every walk. Default true. */ /** Cache Discord-hosted image previews after every walk. Default true. */
previews?: boolean; previews?: boolean;
/** Read dimensions from a downloaded file. Defaults to ffprobe. */
probe?: (path: string) => Promise<{ width: number; height: number } | null>;
} }
interface QueuedJob { interface QueuedJob {
@@ -244,6 +252,9 @@ export class ScrapeEngine {
meta.lastError = error instanceof Error ? error.message : String(error); meta.lastError = error instanceof Error ? error.message : String(error);
this.log(`${meta.id}: failed: ${meta.lastError}`); this.log(`${meta.id}: failed: ${meta.lastError}`);
} finally { } finally {
if ((meta.state === "done" || meta.state === "failed") && !meta.finishedAt) {
meta.finishedAt = new Date(this.now()).toISOString();
}
this.persist(dir, meta); this.persist(dir, meta);
store.close(); store.close();
this.active.set(meta.id, meta); this.active.set(meta.id, meta);
@@ -273,6 +284,11 @@ export class ScrapeEngine {
bytes: outcome.bytes ?? 0 bytes: outcome.bytes ?? 0
}); });
meta.downloaded = (meta.downloaded ?? 0) + 1; meta.downloaded = (meta.downloaded ?? 0) + 1;
if (row.width === null || row.height === null) {
const dimensions = await (this.options.probe ?? probeDimensions)(join(dir, outcome.path));
if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height);
}
} else { } else {
store.markDownloadError(row.id, outcome.error ?? outcome.state); store.markDownloadError(row.id, outcome.error ?? outcome.state);
if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1; if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1;
@@ -283,11 +299,10 @@ export class ScrapeEngine {
} }
} }
/** Cache small previews. In download mode this includes embed thumbnails. */ /** Cache small previews. Embed thumbnails are cached in both modes. */
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> { private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const embedPreviews = meta.mode === "download"; const options = { ...(this.options.download ?? {}), embedPreviews: true };
const options = { ...(this.options.download ?? {}), embedPreviews }; const pending = store.pendingPreviews(5000, true);
const pending = store.pendingPreviews(5000, embedPreviews);
for (const row of pending) { for (const row of pending) {
if (this.stopRequested.has(meta.id)) { if (this.stopRequested.has(meta.id)) {
+2 -2
View File
@@ -76,8 +76,8 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
provider, provider,
downloaded: false, downloaded: false,
size: null, size: null,
width: null, width: embed.width ?? null,
height: null, height: embed.height ?? null,
durationSecs: null durationSecs: null
}; };
+50
View File
@@ -0,0 +1,50 @@
/*
* Query presets for the media filters: aspect ratio and resolution buckets.
* Pure; the SQL fragments are fixed strings, never built from user input.
*/
export const ASPECT_PRESETS = ["vertical", "square", "wide43", "wide169", "ultrawide", "other"] as const;
export type AspectPreset = (typeof ASPECT_PRESETS)[number];
/** Aspect-ratio buckets: vertical/shorts, square, 4:3, 16:9, 21:9, everything else. */
export function aspectCondition(preset: string): string | null {
switch (preset) {
case "vertical":
return "aspect_ratio < 0.95";
case "square":
return "aspect_ratio >= 0.95 AND aspect_ratio < 1.05";
case "wide43":
return "aspect_ratio >= 1.05 AND aspect_ratio < 1.55";
case "wide169":
return "aspect_ratio >= 1.55 AND aspect_ratio < 2.05";
case "ultrawide":
return "aspect_ratio >= 2.05 AND aspect_ratio < 2.8";
case "other":
return "(aspect_ratio IS NULL OR aspect_ratio >= 2.8)";
default:
return null;
}
}
export const RESOLUTION_PRESETS = ["sd", "hd", "fhd", "qhd", "uhd", "unknown"] as const;
export type ResolutionPreset = (typeof RESOLUTION_PRESETS)[number];
/** Resolution buckets by the shortest side, so vertical 1080x1920 counts as FHD. */
export function resolutionCondition(preset: string): string | null {
switch (preset) {
case "sd":
return "MIN(width, height) <= 480";
case "hd":
return "MIN(width, height) > 480 AND MIN(width, height) <= 900";
case "fhd":
return "MIN(width, height) > 900 AND MIN(width, height) <= 1260";
case "qhd":
return "MIN(width, height) > 1260 AND MIN(width, height) <= 1800";
case "uhd":
return "MIN(width, height) > 1800";
case "unknown":
return "(width IS NULL OR height IS NULL)";
default:
return null;
}
}
+8
View File
@@ -5,6 +5,14 @@
export { extractMedia, type ExtractedMedia } from "./extract.js"; export { extractMedia, type ExtractedMedia } from "./extract.js";
export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js"; export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
export {
ASPECT_PRESETS,
RESOLUTION_PRESETS,
aspectCondition,
resolutionCondition,
type AspectPreset,
type ResolutionPreset
} from "./filter-presets.js";
export { export {
downloadGapMs, downloadGapMs,
downloadOne, downloadOne,
+72 -2
View File
@@ -8,6 +8,7 @@ import { join } from "node:path";
import { DatabaseSync } from "node:sqlite"; import { DatabaseSync } from "node:sqlite";
import { fileURLToPath } from "node:url"; import { fileURLToPath } from "node:url";
import { extractMedia } from "./extract.js"; import { extractMedia } from "./extract.js";
import { aspectCondition, resolutionCondition } from "./filter-presets.js";
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js"; import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */ /** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
@@ -77,6 +78,12 @@ export function dirSizeBytes(dir: string): number {
return total; return total;
} }
/** Width / height, rounded to three decimals. Null when either side is missing or zero. */
export function aspectRatioOf(width: number | null | undefined, height: number | null | undefined): number | null {
if (!Number.isFinite(width) || !Number.isFinite(height) || !width || !height) return null;
return Math.round(((width as number) / (height as number)) * 1000) / 1000;
}
export function createScrapeDirs(root: string, id: string): string { export function createScrapeDirs(root: string, id: string): string {
const dir = scrapeDir(root, id); const dir = scrapeDir(root, id);
mkdirSync(join(dir, "media"), { recursive: true }); mkdirSync(join(dir, "media"), { recursive: true });
@@ -99,12 +106,16 @@ export interface MediaRow {
id: number; id: number;
kind: MediaKind; kind: MediaKind;
source: string | null; source: string | null;
author_id: string | null;
author_name: string | null;
posted_at: string | null;
url: string; url: string;
provider: string | null; provider: string | null;
content_type: string | null; content_type: string | null;
size: number | null; size: number | null;
width: number | null; width: number | null;
height: number | null; height: number | null;
aspect_ratio: number | null;
duration_secs: number | null; duration_secs: number | null;
downloaded: number; downloaded: number;
preview_url: string | null; preview_url: string | null;
@@ -131,12 +142,16 @@ CREATE TABLE IF NOT EXISTS media (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL, kind TEXT NOT NULL,
source TEXT, source TEXT,
author_id TEXT,
author_name TEXT,
posted_at TEXT,
url TEXT NOT NULL UNIQUE, url TEXT NOT NULL UNIQUE,
provider TEXT, provider TEXT,
content_type TEXT, content_type TEXT,
size INTEGER, size INTEGER,
width INTEGER, width INTEGER,
height INTEGER, height INTEGER,
aspect_ratio REAL,
duration_secs REAL, duration_secs REAL,
downloaded INTEGER NOT NULL DEFAULT 0, downloaded INTEGER NOT NULL DEFAULT 0,
preview_url TEXT, preview_url TEXT,
@@ -157,6 +172,7 @@ CREATE TABLE IF NOT EXISTS media_sources (
); );
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind); CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
CREATE INDEX IF NOT EXISTS idx_media_posted ON media(posted_at);
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id); CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
`; `;
@@ -178,9 +194,13 @@ export class ScrapeStore {
if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER"); if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER"); if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT"); if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
if (!names.has("author_id")) this.db.exec("ALTER TABLE media ADD COLUMN author_id TEXT");
if (!names.has("author_name")) this.db.exec("ALTER TABLE media ADD COLUMN author_name TEXT");
if (!names.has("posted_at")) this.db.exec("ALTER TABLE media ADD COLUMN posted_at TEXT");
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT"); if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT"); if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT"); if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
} }
close(): void { close(): void {
@@ -194,7 +214,7 @@ export class ScrapeStore {
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)" "INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
); );
const insertMedia = this.db.prepare( const insertMedia = this.db.prepare(
"INSERT OR IGNORE INTO media (kind, source, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" "INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
); );
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?"); const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
const linkSource = this.db.prepare( const linkSource = this.db.prepare(
@@ -217,12 +237,16 @@ export class ScrapeStore {
const result = insertMedia.run( const result = insertMedia.run(
item.kind, item.kind,
item.source, item.source,
message.author.id,
message.author.username,
message.timestamp,
item.url, item.url,
item.provider, item.provider,
item.contentType, item.contentType,
item.size, item.size,
item.width, item.width,
item.height, item.height,
aspectRatioOf(item.width, item.height),
item.durationSecs, item.durationSecs,
item.downloaded ? 1 : 0, item.downloaded ? 1 : 0,
item.previewUrl, item.previewUrl,
@@ -321,6 +345,23 @@ export class ScrapeStore {
this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id); this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
} }
/** Fill in dimensions after a download probe. */
updateDimensions(id: number, width: number, height: number): void {
this.db
.prepare("UPDATE media SET width = ?, height = ?, aspect_ratio = ? WHERE id = ?")
.run(width, height, aspectRatioOf(width, height), id);
}
/** Distinct posters with their item counts, biggest first. */
authors(limit = 200): Array<{ id: string | null; name: string | null; count: number }> {
const rows = this.db
.prepare(
"SELECT author_id AS id, author_name AS name, COUNT(*) AS count FROM media WHERE author_name IS NOT NULL GROUP BY author_id, author_name ORDER BY count DESC LIMIT ?"
)
.all(limit) as unknown as Array<{ id: string | null; name: string | null; count: number }>;
return rows;
}
getMedia(id: number): MediaRow | null { getMedia(id: number): MediaRow | null {
const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
| MediaRow | MediaRow
@@ -329,13 +370,24 @@ export class ScrapeStore {
} }
queryMedia( queryMedia(
options: { kind?: MediaKind; source?: string; limit?: number; offset?: number } = {} options: {
kind?: MediaKind;
source?: string;
authorId?: string;
aspect?: string;
resolution?: string;
after?: string;
before?: string;
limit?: number;
offset?: number;
} = {}
): MediaRow[] { ): MediaRow[] {
const limit = Math.max(1, Math.min(options.limit ?? 50, 500)); const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
const offset = Math.max(0, options.offset ?? 0); const offset = Math.max(0, options.offset ?? 0);
const conditions: string[] = []; const conditions: string[] = [];
const values: Array<string | number> = []; const values: Array<string | number> = [];
if (options.kind) { if (options.kind) {
conditions.push("kind = ?"); conditions.push("kind = ?");
values.push(options.kind); values.push(options.kind);
@@ -344,6 +396,24 @@ export class ScrapeStore {
conditions.push("source = ?"); conditions.push("source = ?");
values.push(options.source); values.push(options.source);
} }
if (options.authorId) {
conditions.push("author_id = ?");
values.push(options.authorId);
}
if (options.after) {
conditions.push("posted_at >= ?");
values.push(options.after);
}
if (options.before) {
conditions.push("posted_at <= ?");
values.push(options.before);
}
const aspect = options.aspect ? aspectCondition(options.aspect) : null;
if (aspect) conditions.push(aspect);
const resolution = options.resolution ? resolutionCondition(options.resolution) : null;
if (resolution) conditions.push(resolution);
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
const rows = this.db const rows = this.db
+5
View File
@@ -39,6 +39,9 @@ export interface CoreEmbed {
videoUrl: string | null; videoUrl: string | null;
imageUrl: string | null; imageUrl: string | null;
thumbnailUrl: string | null; thumbnailUrl: string | null;
/** Dimensions of the video, image, or thumbnail, first one Discord provides. */
width?: number | null;
height?: number | null;
} }
export interface CoreSticker { export interface CoreSticker {
@@ -89,6 +92,8 @@ export interface ScrapeMeta {
downloadErrors?: number; downloadErrors?: number;
/** Cached image previews. */ /** Cached image previews. */
previewed?: number; previewed?: number;
/** When the scrape reached done or failed. */
finishedAt?: string | null;
} }
export function emptyMediaCounts(): Record<MediaKind, number> { export function emptyMediaCounts(): Record<MediaKind, number> {
+80 -5
View File
@@ -12,6 +12,7 @@ import { downloadOne, downloadPreview, previewUrlFor } from "../src/download.js"
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js"; import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
import { extractMedia } from "../src/extract.js"; import { extractMedia } from "../src/extract.js";
import { PuppetBackend } from "../src/puppet-backend.js"; import { PuppetBackend } from "../src/puppet-backend.js";
import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
import { isNearLimit, planRetention } from "../src/retention.js"; import { isNearLimit, planRetention } from "../src/retention.js";
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js"; import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
import type { CoreAttachment, CoreMessage } from "../src/types.js"; import type { CoreAttachment, CoreMessage } from "../src/types.js";
@@ -196,6 +197,7 @@ void (async () => {
assert.equal(meta?.messages, 3); assert.equal(meta?.messages, 3);
assert.equal(meta?.cursor, "99"); assert.equal(meta?.cursor, "99");
assert.equal(meta?.media.image, 1); assert.equal(meta?.media.image, 1);
assert.ok(meta?.finishedAt);
assert.equal(backend.calls[0].limit, 2); assert.equal(backend.calls[0].limit, 2);
assert.equal(backend.calls[1].before, "100"); assert.equal(backend.calls[1].before, "100");
@@ -209,6 +211,9 @@ void (async () => {
assert.equal(stored.queryMedia({ kind: "image" }).length, 1); assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
assert.equal(stored.queryMedia({ kind: "video" }).length, 0); assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
assert.equal(stored.storedBytesByKind().image, 0); assert.equal(stored.storedBytesByKind().image, 0);
const row = stored.queryMedia({ kind: "image" })[0];
assert.equal(row.author_name, "tester");
assert.equal(row.posted_at, "2026-09-16T00:00:00.000Z");
stored.close(); stored.close();
assert.equal(listScrapes(root).length, 1); assert.equal(listScrapes(root).length, 1);
@@ -405,14 +410,30 @@ void (async () => {
assert.equal(existsSync(join(root, started.id, row.file_path)), true); assert.equal(existsSync(join(root, started.id, row.file_path)), true);
}); });
await test("store filters by source and kind", () => { await test("aspect and resolution presets have fixed buckets", () => {
assert.equal(aspectCondition("vertical"), "aspect_ratio < 0.95");
assert.equal(aspectCondition("wide169"), "aspect_ratio >= 1.55 AND aspect_ratio < 2.05");
assert.equal(aspectCondition("nope"), null);
assert.equal(resolutionCondition("fhd"), "MIN(width, height) > 900 AND MIN(width, height) <= 1260");
assert.equal(resolutionCondition("unknown"), "(width IS NULL OR height IS NULL)");
assert.equal(resolutionCondition("nope"), null);
});
await test("store filters by source, kind, author, ratio, resolution, and dates", () => {
const dir = join(root, "store-source"); const dir = join(root, "store-source");
mkdirSync(dir, { recursive: true }); mkdirSync(dir, { recursive: true });
const store = new ScrapeStore(dir); const store = new ScrapeStore(dir);
store.insertPage([ store.insertPage([
message("700", { message("700", {
attachments: [attachment({ filename: "pic.png", contentType: "image/png" })], attachments: [
attachment({
filename: "pic.png",
contentType: "image/png",
width: 640,
height: 360
})
],
embeds: [ embeds: [
{ {
type: "video", type: "video",
@@ -432,6 +453,59 @@ void (async () => {
assert.equal(store.queryMedia({ source: "sticker" }).length, 0); assert.equal(store.queryMedia({ source: "sticker" }).length, 0);
assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1); assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1);
assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0); assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0);
assert.equal(store.queryMedia({ aspect: "wide169" }).length, 1);
assert.equal(store.queryMedia({ aspect: "vertical" }).length, 0);
assert.equal(store.queryMedia({ resolution: "sd" }).length, 1);
assert.equal(store.queryMedia({ resolution: "fhd" }).length, 0);
assert.equal(store.queryMedia({ authorId: "1" }).length, 2);
assert.equal(store.queryMedia({ authorId: "999" }).length, 0);
assert.equal(store.queryMedia({ after: "2026-09-16T00:00:00.000Z" }).length, 2);
assert.equal(store.queryMedia({ before: "2026-09-15T00:00:00.000Z" }).length, 0);
const [wide] = store.queryMedia({ kind: "image" });
assert.equal(wide.aspect_ratio, 1.778);
store.close();
});
await test("engine probes dimensions when the metadata lacks them", async () => {
const backend = new FakeBackend([
{
messages: [
message("701", {
attachments: [attachment({ filename: "probe.png", contentType: "image/png" })]
})
]
}
]);
const engine = new ScrapeEngine({
rootDir: root,
pageSize: 2,
pageMinMs: 0,
pageMaxMs: 0,
sleep: async () => {},
random: () => 0,
log: () => {},
previews: false,
probe: async () => ({ width: 1080, height: 1920 }),
download: {
fetchImpl: async () =>
new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
minGapMs: 0,
maxGapMs: 0,
sleep: async () => {}
}
});
const started = await engine.start({ channelId: "129", guildId: null, mode: "download", backend });
await engine.processQueue();
const store = new ScrapeStore(join(root, started.id));
const row = store.queryMedia({ kind: "image" })[0];
assert.equal(row.width, 1080);
assert.equal(row.height, 1920);
assert.equal(row.aspect_ratio, 0.563);
store.close(); store.close();
}); });
@@ -502,7 +576,7 @@ void (async () => {
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false); assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
}); });
await test("engine caches previews in links mode and skips embeds", async () => { await test("engine caches previews in links mode, including embed thumbnails", async () => {
const backend = new FakeBackend([ const backend = new FakeBackend([
{ {
messages: [ messages: [
@@ -551,8 +625,9 @@ void (async () => {
const meta = engine.get(started.id); const meta = engine.get(started.id);
assert.equal(meta?.state, "done"); assert.equal(meta?.state, "done");
assert.equal(meta?.previewed, 1); assert.equal(meta?.previewed, 2);
assert.equal(fetched.some(url => url.includes("ytimg.com")), false); assert.equal(fetched.some(url => url.includes("ytimg.com")), true);
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
const store = new ScrapeStore(join(root, started.id)); const store = new ScrapeStore(join(root, started.id));
const row = store.queryMedia({ kind: "image" })[0]; const row = store.queryMedia({ kind: "image" })[0];
+5 -1
View File
@@ -30,7 +30,11 @@ async function request(path, options) {
} catch { } catch {
throw new Error(`request failed (${response.status})`); throw new Error(`request failed (${response.status})`);
} }
if (!body.ok) throw new Error(body.error ?? `request failed (${response.status})`); if (!body.ok) {
const error = new Error(body.error ?? `request failed (${response.status})`);
if (body.code) error.code = body.code;
throw error;
}
return body.result; return body.result;
} }
+78
View File
@@ -202,9 +202,87 @@
box-shadow: inset 0 0 0 1px var(--color-neutral-800); box-shadow: inset 0 0 0 1px var(--color-neutral-800);
} }
.vk-media-modal-box.vk-media-wide {
width: min(760px, 94vw);
height: min(80vh, 720px);
}
.vk-media-modal-list { .vk-media-modal-list {
max-height: 220px; max-height: 220px;
overflow-y: auto; overflow-y: auto;
font: 400 11px/1.5 ui-monospace, Menlo, monospace; font: 400 11px/1.5 ui-monospace, Menlo, monospace;
color: var(--color-neutral-400); color: var(--color-neutral-400);
} }
.vk-media-wide .vk-media-modal-list {
max-height: none;
flex: 1;
}
.vk-media-history-row {
display: flex;
flex-direction: column;
gap: 2px;
padding: 7px 0;
border-bottom: 1px solid var(--color-neutral-900);
}
.vk-media-tree {
display: flex;
flex-direction: column;
gap: 6px;
flex: 1;
overflow-y: auto;
}
.vk-media-tree-head {
padding: 6px 8px;
border: none;
border-radius: var(--radius-sm, 6px);
background: var(--color-neutral-900);
color: var(--color-neutral-200);
font: 500 12px/1.2 inherit;
text-align: left;
cursor: pointer;
}
.vk-media-tree-children {
display: flex;
flex-direction: column;
gap: 3px;
margin: 4px 0 4px 14px;
}
.vk-media-channel {
display: flex;
align-items: center;
gap: 6px;
}
.vk-media-channel-name {
flex: 1;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font: 400 11px/1.4 ui-monospace, Menlo, monospace;
color: var(--color-neutral-300);
}
@keyframes vk-media-pulse {
0%,
100% {
opacity: 0.25;
}
50% {
opacity: 1;
}
}
.vk-media-pulse {
flex: none;
width: 8px;
height: 8px;
border-radius: 50%;
background: var(--color-accent-400, #4f8cff);
animation: vk-media-pulse 1.2s ease-in-out infinite;
}
+328 -29
View File
@@ -18,8 +18,19 @@
items: [], items: [],
scrapes: [], scrapes: [],
channels: [], channels: [],
guilds: [],
dms: [],
loadedChannels: false, loadedChannels: false,
filters: { source: "", channelId: "" } authors: null,
filters: {
source: "",
channelId: "",
authorId: "",
aspect: "",
resolution: "",
after: "",
before: ""
}
}; };
const overlay = document.createElement("div"); const overlay = document.createElement("div");
@@ -95,8 +106,9 @@
if (state.loadedChannels) return; if (state.loadedChannels) return;
try { try {
const data = await window.VK.api.get("/api/state"); const data = await window.VK.api.get("/api/state");
state.guilds = data?.guilds ?? [];
state.channels = []; state.channels = [];
for (const guild of data?.guilds ?? []) { for (const guild of state.guilds) {
for (const channel of guild.channels ?? []) { for (const channel of guild.channels ?? []) {
if (channel.type !== 0 && channel.type !== 5) continue; if (channel.type !== 0 && channel.type !== 5) continue;
state.channels.push({ state.channels.push({
@@ -109,8 +121,9 @@
// DMs and group DMs: the bot cannot see them, so they always use the puppet. // DMs and group DMs: the bot cannot see them, so they always use the puppet.
try { try {
const dms = await window.VK.api.get("/api/dms?limit=50"); const dmData = await window.VK.api.get("/api/dms?limit=50");
for (const dm of dms ?? []) { state.dms = dmData?.channels ?? [];
for (const dm of state.dms) {
const names = (dm.recipients ?? []) const names = (dm.recipients ?? [])
.map(recipient => recipient.username) .map(recipient => recipient.username)
.filter(Boolean) .filter(Boolean)
@@ -122,15 +135,25 @@
}); });
} }
} catch { } catch {
// no cached DMs yet state.dms = [];
} }
state.loadedChannels = true; state.loadedChannels = true;
} catch { } catch {
state.guilds = [];
state.channels = []; state.channels = [];
} }
} }
async function loadAuthors() {
if (state.authors) return;
try {
state.authors = await window.VK.api.get("/api/media/authors");
} catch {
state.authors = [];
}
}
async function loadScrapes() { async function loadScrapes() {
try { try {
state.scrapes = await window.VK.api.get("/api/media/scrapes"); state.scrapes = await window.VK.api.get("/api/media/scrapes");
@@ -151,8 +174,13 @@
const kind = state.kind === "all" ? "" : `&kind=${state.kind}`; const kind = state.kind === "all" ? "" : `&kind=${state.kind}`;
const source = state.filters.source ? `&source=${state.filters.source}` : ""; const source = state.filters.source ? `&source=${state.filters.source}` : "";
const channel = state.filters.channelId ? `&channelId=${state.filters.channelId}` : ""; const channel = state.filters.channelId ? `&channelId=${state.filters.channelId}` : "";
const author = state.filters.authorId ? `&authorId=${state.filters.authorId}` : "";
const aspect = state.filters.aspect ? `&aspect=${state.filters.aspect}` : "";
const resolution = state.filters.resolution ? `&resolution=${state.filters.resolution}` : "";
const after = state.filters.after ? `&after=${state.filters.after}T00:00:00.000Z` : "";
const before = state.filters.before ? `&before=${state.filters.before}T23:59:59.999Z` : "";
const items = await window.VK.api.get( const items = await window.VK.api.get(
`/api/media?limit=60&offset=${state.offset}${kind}${source}${channel}` `/api/media?limit=60&offset=${state.offset}${kind}${source}${channel}${author}${aspect}${resolution}${after}${before}`
); );
state.items = state.items.concat(items); state.items = state.items.concat(items);
state.offset += items.length; state.offset += items.length;
@@ -163,22 +191,27 @@
renderGrid(); renderGrid();
} }
async function startScrape(mode) { async function startScrape(channelId, guildId, mode) {
const select = overlay.querySelector("#media-channel"); if (!channelId) {
if (!select?.value) { setStatus("no channel selected");
setStatus("no text channel available to scrape");
return; return;
} }
const channel = state.channels.find(entry => entry.id === select.value);
try { try {
const scrape = await window.VK.api.post("/api/media/scrape", { const scrape = await window.VK.api.post("/api/media/scrape", {
channelId: select.value, channelId,
guildId: channel?.guildId ?? null, guildId: guildId ?? null,
mode mode,
backend: guildId ? "auto" : "puppet"
}); });
setStatus(`started ${scrape.id} (${mode})`); setStatus(`started ${scrape.id} (${mode})`);
scrapeModal.hidden = true;
} catch (error) { } catch (error) {
setStatus(`start failed: ${error.message}`); if (error.code === "MEDIA_LIMIT" || String(error.message).includes("MEDIA_LIMIT")) {
setStatus(`near the storage limit: ${error.message}`);
} else {
setStatus(`start failed: ${error.message}`);
}
} }
void loadScrapes(); void loadScrapes();
} }
@@ -189,7 +222,23 @@
const rows = active.length ? active : state.scrapes.slice(0, 5); const rows = active.length ? active : state.scrapes.slice(0, 5);
for (const scrape of rows) { for (const scrape of rows) {
const row = el("div", "vk-media-scrape"); const row = el("div", "vk-media-scrape");
row.append(el("span", null, `${scrape.channelId} · ${scrape.state} · ${scrape.pages}p · ${scrape.messages}m`)); const media = scrape.media ?? {};
if (scrape.state === "running" || scrape.state === "queued") {
row.append(el("span", "vk-media-pulse"));
}
row.append(
el(
"span",
null,
`${channelLabel(scrape.channelId)} · ${scrape.state} · ${scrape.pages}p · ${scrape.messages}m · ` +
`img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0}` +
(scrape.downloaded ? ` · files ${scrape.downloaded}` : "") +
(scrape.previewed ? ` · previews ${scrape.previewed}` : "")
)
);
if (active.includes(scrape)) { if (active.includes(scrape)) {
row.append( row.append(
button("Stop", async () => { button("Stop", async () => {
@@ -197,7 +246,10 @@
void loadScrapes(); void loadScrapes();
}) })
); );
} else {
row.append(el("span", "vk-media-status", durationLabel(scrape)));
} }
scrapeList.append(row); scrapeList.append(row);
} }
} }
@@ -211,6 +263,11 @@
card.target = "_blank"; card.target = "_blank";
card.rel = "noreferrer"; card.rel = "noreferrer";
const postedBy = [item.authorName, item.postedAt ? new Date(item.postedAt).toLocaleString() : null]
.filter(Boolean)
.join(" · ");
if (postedBy) card.title = postedBy;
if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) { if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) {
const img = document.createElement("img"); const img = document.createElement("img");
img.loading = "lazy"; img.loading = "lazy";
@@ -225,6 +282,29 @@
} }
} }
function filterSelect(options, value, onChange) {
const select = document.createElement("select");
select.className = "vk-chip vk-chip-off";
for (const [optionValue, label] of options) {
const option = document.createElement("option");
option.value = optionValue;
option.textContent = label;
select.append(option);
}
select.value = value;
select.addEventListener("change", () => onChange(select.value));
return select;
}
function dateInput(value, onChange) {
const input = document.createElement("input");
input.type = "date";
input.className = "vk-media-input";
input.value = value ?? "";
input.addEventListener("change", () => onChange(input.value));
return input;
}
function renderToolbar() { function renderToolbar() {
toolbar.replaceChildren(); toolbar.replaceChildren();
if (state.view === "media") { if (state.view === "media") {
@@ -239,19 +319,8 @@
toolbar.append(chip); toolbar.append(chip);
} }
const select = document.createElement("select"); toolbar.append(button("New scrape…", openScrapeModal));
select.id = "media-channel"; toolbar.append(button("History", openHistoryModal));
select.className = "vk-chip vk-chip-off";
for (const channel of state.channels) {
const option = document.createElement("option");
option.value = channel.id;
option.textContent = channel.label;
select.append(option);
}
toolbar.append(select);
toolbar.append(button("Start links", () => void startScrape("links")));
toolbar.append(button("Start download", () => void startScrape("download")));
const sourceSelect = document.createElement("select"); const sourceSelect = document.createElement("select");
sourceSelect.className = "vk-chip vk-chip-off"; sourceSelect.className = "vk-chip vk-chip-off";
@@ -296,6 +365,69 @@
void loadMedia(true); void loadMedia(true);
}); });
toolbar.append(channelSelect); toolbar.append(channelSelect);
const authorOptions = [["", "Any user"]].concat(
(state.authors ?? [])
.filter(author => author.id)
.map(author => [author.id, `${author.name} (${author.count})`])
);
toolbar.append(
filterSelect(authorOptions, state.filters.authorId, value => {
state.filters.authorId = value;
void loadMedia(true);
})
);
toolbar.append(
filterSelect(
[
["", "Any ratio"],
["vertical", "Vertical"],
["square", "1:1"],
["wide43", "4:3"],
["wide169", "16:9"],
["ultrawide", "21:9"],
["other", "Other"]
],
state.filters.aspect,
value => {
state.filters.aspect = value;
void loadMedia(true);
}
)
);
toolbar.append(
filterSelect(
[
["", "Any quality"],
["sd", "SD (<=480p)"],
["hd", "HD (720p)"],
["fhd", "FHD (1080p)"],
["qhd", "QHD (1440p)"],
["uhd", "4K+"],
["unknown", "Unknown"]
],
state.filters.resolution,
value => {
state.filters.resolution = value;
void loadMedia(true);
}
)
);
toolbar.append(
el("span", "vk-media-status", "From"),
dateInput(state.filters.after, value => {
state.filters.after = value;
void loadMedia(true);
}),
el("span", "vk-media-status", "To"),
dateInput(state.filters.before, value => {
state.filters.before = value;
void loadMedia(true);
})
);
} else { } else {
toolbar.append(el("span", null, "Slopbot is the dedicated bot core. Setup: docs/slopbot.md.")); toolbar.append(el("span", null, "Slopbot is the dedicated bot core. Setup: docs/slopbot.md."));
} }
@@ -314,6 +446,7 @@
: "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)"; : "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)";
} else { } else {
void loadChannels().then(renderToolbar); void loadChannels().then(renderToolbar);
void loadAuthors().then(renderToolbar);
void loadMedia(true); void loadMedia(true);
} }
void loadScrapes(); void loadScrapes();
@@ -332,6 +465,172 @@
document.querySelector("#scope-guild")?.addEventListener("click", hide); document.querySelector("#scope-guild")?.addEventListener("click", hide);
document.querySelector("#scope-dm")?.addEventListener("click", hide); document.querySelector("#scope-dm")?.addEventListener("click", hide);
// ---- Scrape popup: server or group-chat trees ----
const scrapeModal = document.createElement("div");
scrapeModal.className = "vk-media-modal";
scrapeModal.hidden = true;
scrapeModal.innerHTML = `
<div class="vk-media-modal-box vk-media-wide">
<strong>New scrape</strong>
<div class="vk-media-toolbar">
<button id="scrape-tab-server" class="vk-chip vk-chip-on" type="button">Server</button>
<button id="scrape-tab-group" class="vk-chip vk-chip-off" type="button">Group chat</button>
<span id="scrape-hint" class="vk-media-status"></span>
</div>
<div id="scrape-tree" class="vk-media-tree"></div>
<div class="vk-media-toolbar">
<button id="scrape-cancel" class="btn btn-secondary vk-small-btn" type="button">Close</button>
</div>
</div>
`;
document.body.append(scrapeModal);
let scrapeTab = "server";
function channelRow(channelId, guildId, label) {
const row = el("div", "vk-media-channel");
row.append(el("span", "vk-media-channel-name", label));
row.append(button("Links", () => void startScrape(channelId, guildId, "links")));
row.append(button("Download", () => void startScrape(channelId, guildId, "download")));
return row;
}
function renderScrapeTree() {
const tree = scrapeModal.querySelector("#scrape-tree");
const hint = scrapeModal.querySelector("#scrape-hint");
tree.replaceChildren();
if (scrapeTab === "group") {
const groups = state.dms.filter(dm => dm.type === 3);
hint.textContent = groups.length ? "Group chats use the puppet account." : "No cached group chats yet.";
for (const dm of groups) {
const names = (dm.recipients ?? [])
.map(recipient => recipient.username)
.filter(Boolean)
.join(", ");
tree.append(channelRow(dm.id, null, dm.name || names || dm.id));
}
return;
}
hint.textContent = state.guilds.length ? "" : "No guilds cached yet.";
for (const guild of state.guilds) {
const group = el("div", "vk-media-tree-guild");
const head = el("button", "vk-media-tree-head", `> ${guild.name}`);
head.type = "button";
const children = el("div", "vk-media-tree-children");
children.hidden = true;
head.addEventListener("click", () => {
children.hidden = !children.hidden;
head.textContent = `${children.hidden ? ">" : "v"} ${guild.name}`;
});
group.append(head, children);
for (const channel of guild.channels ?? []) {
if (channel.type !== 0 && channel.type !== 5) continue;
children.append(channelRow(channel.id, guild.id, `#${channel.name}`));
}
tree.append(group);
}
}
function openScrapeModal() {
renderScrapeTree();
scrapeModal.hidden = false;
void loadChannels().then(renderScrapeTree);
}
for (const [tab, id] of [
["server", "#scrape-tab-server"],
["group", "#scrape-tab-group"]
]) {
scrapeModal.querySelector(id).addEventListener("click", () => {
scrapeTab = tab;
scrapeModal.querySelector("#scrape-tab-server").className =
tab === "server" ? "vk-chip vk-chip-on" : "vk-chip vk-chip-off";
scrapeModal.querySelector("#scrape-tab-group").className =
tab === "group" ? "vk-chip vk-chip-on" : "vk-chip vk-chip-off";
renderScrapeTree();
});
}
scrapeModal.querySelector("#scrape-cancel").addEventListener("click", () => {
scrapeModal.hidden = true;
});
// ---- History popup ----
const historyModal = document.createElement("div");
historyModal.className = "vk-media-modal";
historyModal.hidden = true;
historyModal.innerHTML = `
<div class="vk-media-modal-box vk-media-wide">
<strong>Scrape history</strong>
<div id="history-list" class="vk-media-modal-list"></div>
<div class="vk-media-toolbar">
<button id="history-close" class="btn btn-secondary vk-small-btn" type="button">Close</button>
</div>
</div>
`;
document.body.append(historyModal);
function durationLabel(scrape) {
const start = new Date(scrape.createdAt).getTime();
const end = scrape.finishedAt ? new Date(scrape.finishedAt).getTime() : Date.now();
const seconds = Math.max(0, Math.round((end - start) / 1000));
if (seconds < 60) return `${seconds}s`;
const minutes = Math.floor(seconds / 60);
if (minutes < 60) return `${minutes}m ${seconds % 60}s`;
return `${Math.floor(minutes / 60)}h ${minutes % 60}m`;
}
function channelLabel(channelId) {
return state.channels.find(channel => channel.id === channelId)?.label ?? channelId;
}
async function openHistoryModal() {
const list = historyModal.querySelector("#history-list");
list.replaceChildren(el("div", null, "loading..."));
historyModal.hidden = false;
try {
const [scrapes, storage] = await Promise.all([
window.VK.api.get("/api/media/scrapes"),
window.VK.api.get("/api/media/storage")
]);
await loadChannels();
list.replaceChildren();
const bytesById = new Map((storage?.scrapes ?? []).map(entry => [entry.id, entry.bytes]));
for (const scrape of scrapes) {
const media = scrape.media ?? {};
const row = el("div", "vk-media-history-row");
row.append(el("strong", null, channelLabel(scrape.channelId)));
row.append(
el(
"span",
null,
`${scrape.state} · started ${new Date(scrape.createdAt).toLocaleString()} · ` +
`took ${durationLabel(scrape)} · ${scrape.pages}p/${scrape.messages}m · ` +
`img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0} · ` +
`${formatBytes(bytesById.get(scrape.id) ?? 0)} · files ${scrape.downloaded ?? 0} · previews ${scrape.previewed ?? 0}`
)
);
list.append(row);
}
if (!scrapes.length) list.append(el("div", null, "No scrapes yet."));
} catch (error) {
list.replaceChildren(el("div", null, `history failed: ${error.message}`));
}
}
historyModal.querySelector("#history-close").addEventListener("click", () => {
historyModal.hidden = true;
});
// ---- Settings: media storage ---- // ---- Settings: media storage ----
function formatBytes(bytes) { function formatBytes(bytes) {
+27 -2
View File
@@ -829,12 +829,37 @@ function buildServer(): McpServer {
.describe("Where the item came from"), .describe("Where the item came from"),
scrapeId: z.string().optional().describe("Restrict to one scrape"), scrapeId: z.string().optional().describe("Restrict to one scrape"),
channelId: z.string().optional().describe("Restrict to one source channel"), channelId: z.string().optional().describe("Restrict to one source channel"),
authorId: z.string().optional().describe("Restrict to one poster"),
aspect: z
.enum(["vertical", "square", "wide43", "wide169", "ultrawide", "other"])
.optional()
.describe("Aspect-ratio bucket"),
resolution: z
.enum(["sd", "hd", "fhd", "qhd", "uhd", "unknown"])
.optional()
.describe("Resolution bucket by shortest side"),
after: z.string().optional().describe("Only items posted at or after this ISO time"),
before: z.string().optional().describe("Only items posted at or before this ISO time"),
limit: z.number().int().min(1).max(500).optional(), limit: z.number().int().min(1).max(500).optional(),
offset: z.number().int().min(0).optional() offset: z.number().int().min(0).optional()
} }
}, },
async ({ kind, source, scrapeId, channelId, limit, offset }) => async ({ kind, source, scrapeId, channelId, authorId, aspect, resolution, after, before, limit, offset }) =>
jsonText(media.queryMedia({ kind, source, scrapeId, channelId, limit, offset })) jsonText(
media.queryMedia({
kind,
source,
scrapeId,
channelId,
authorId,
aspect,
resolution,
after,
before,
limit,
offset
})
)
); );
server.registerTool( server.registerTool(
+45
View File
@@ -93,6 +93,12 @@ export interface MediaItemResult {
channelId: string; channelId: string;
kind: MediaKind; kind: MediaKind;
source: string | null; source: string | null;
authorId: string | null;
authorName: string | null;
postedAt: string | null;
width: number | null;
height: number | null;
aspectRatio: number | null;
url: string; url: string;
localUrl: string | null; localUrl: string | null;
thumbUrl: string | null; thumbUrl: string | null;
@@ -337,6 +343,29 @@ export class MediaService {
}; };
} }
/** Distinct posters across every scrape, biggest first. */
listAuthors(): Array<{ id: string | null; name: string | null; count: number }> {
const totals = new Map<string, { id: string | null; name: string | null; count: number }>();
for (const meta of this.engine.list()) {
if (meta.state === "deleted") continue;
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
try {
for (const author of store.authors()) {
const key = author.id ?? author.name ?? "unknown";
const current = totals.get(key);
if (current) current.count += author.count;
else totals.set(key, { ...author });
}
} finally {
store.close();
}
}
return [...totals.values()].sort((a, b) => b.count - a.count);
}
/** One media row inside a scrape, with its scrape and channel for context. */ /** One media row inside a scrape, with its scrape and channel for context. */
getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null { getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null {
const meta = this.engine.get(scrapeId); const meta = this.engine.get(scrapeId);
@@ -358,6 +387,11 @@ export class MediaService {
source?: string; source?: string;
scrapeId?: string; scrapeId?: string;
channelId?: string; channelId?: string;
authorId?: string;
aspect?: string;
resolution?: string;
after?: string;
before?: string;
limit?: number; limit?: number;
offset?: number; offset?: number;
} = {} } = {}
@@ -377,6 +411,11 @@ export class MediaService {
const rows = store.queryMedia({ const rows = store.queryMedia({
kind: options.kind, kind: options.kind,
source: options.source, source: options.source,
authorId: options.authorId,
aspect: options.aspect,
resolution: options.resolution,
after: options.after,
before: options.before,
limit: limit + offset limit: limit + offset
}); });
for (const row of rows) { for (const row of rows) {
@@ -388,6 +427,12 @@ export class MediaService {
channelId: meta.channelId, channelId: meta.channelId,
kind: row.kind, kind: row.kind,
source: row.source, source: row.source,
authorId: row.author_id,
authorName: row.author_name,
postedAt: row.posted_at,
width: row.width,
height: row.height,
aspectRatio: row.aspect_ratio,
url: row.url, url: row.url,
localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null, localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null,
thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null, thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
+27 -1
View File
@@ -799,10 +799,36 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
? String(req.query.source) ? String(req.query.source)
: undefined; : undefined;
const channelId = req.query.channelId ? String(req.query.channelId) : undefined; const channelId = req.query.channelId ? String(req.query.channelId) : undefined;
const authorId = req.query.authorId ? String(req.query.authorId) : undefined;
const aspect = req.query.aspect ? String(req.query.aspect) : undefined;
const resolution = req.query.resolution ? String(req.query.resolution) : undefined;
const after = req.query.after ? String(req.query.after) : undefined;
const before = req.query.before ? String(req.query.before) : undefined;
const limit = clampLimit(req.query.limit, 60, 500); const limit = clampLimit(req.query.limit, 60, 500);
const offset = Math.max(0, Number(req.query.offset ?? 0) || 0); const offset = Math.max(0, Number(req.query.offset ?? 0) || 0);
res.json({ ok: true, result: deps.media.queryMedia({ kind, source, channelId, limit, offset }) }); res.json({
ok: true,
result: deps.media.queryMedia({
kind,
source,
channelId,
authorId,
aspect,
resolution,
after,
before,
limit,
offset
})
});
})
);
app.get(
"/api/media/authors",
wrap(async (_req, res) => {
res.json({ ok: true, result: deps.media.listAuthors() });
}) })
); );
+25 -9
View File
@@ -210,16 +210,32 @@ function mapAttachments(message: any) {
})); }));
} }
/** Dimensions of the first embed part that has them: video, image, then thumbnail. */
function embedSize(embed: any) {
for (const part of [embed?.video, embed?.image, embed?.thumbnail]) {
if (part && Number.isFinite(part.width) && Number.isFinite(part.height)) {
return { width: part.width, height: part.height };
}
}
return { width: null, height: null };
}
function mapEmbeds(message: any) { function mapEmbeds(message: any) {
return (message?.embeds ?? []).map((embed: any) => ({ return (message?.embeds ?? []).map((embed: any) => {
type: embed?.type ?? null, const size = embedSize(embed);
url: embed?.url ?? null,
title: embed?.title ?? null, return {
provider: embed?.provider?.name ?? null, type: embed?.type ?? null,
videoUrl: embedUrl(embed?.video), url: embed?.url ?? null,
imageUrl: embedUrl(embed?.image), title: embed?.title ?? null,
thumbnailUrl: embedUrl(embed?.thumbnail) provider: embed?.provider?.name ?? null,
})); videoUrl: embedUrl(embed?.video),
imageUrl: embedUrl(embed?.image),
thumbnailUrl: embedUrl(embed?.thumbnail),
width: size.width,
height: size.height
};
});
} }
function mapStickers(message: any) { function mapStickers(message: any) {