Add links-mode embed thumbnails, item metadata, and media filters
This commit is contained in:
@@ -40,6 +40,15 @@ mobile draft at 390px.
|
|||||||
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
|
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
|
||||||
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
|
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
|
||||||
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
|
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
|
||||||
|
- [x] C14 Links-mode embed thumbnails, item metadata, filters, and the scrape popup. Metadata:
|
||||||
|
poster (`author_id`/`author_name`), `posted_at`, `width`/`height`, `aspect_ratio`, with an
|
||||||
|
`ffprobe` fallback after downloads. Filters: poster, aspect preset (vertical/square/4:3/16:9/
|
||||||
|
21:9/other), resolution preset (SD/HD/FHD/QHD/4K+/unknown, by shortest side), and a
|
||||||
|
posted-date range. UI: scrape popup with Server and Group-chat trees (server -> channels, each
|
||||||
|
with Links/Download), History popup (time, duration, pages/messages, media, bytes, files,
|
||||||
|
previews), live progress pulse, full-window view. Live run `...-20260916094120`: 11 previews
|
||||||
|
in links mode (embed thumbnails included), poster zzib, a 9:16 short stored as 0.563, and the
|
||||||
|
16:9 filter matched.
|
||||||
- [x] C13 Embed thumbnails in download mode: embed rows cache their stored thumbnail URL;
|
- [x] C13 Embed thumbnails in download mode: embed rows cache their stored thumbnail URL;
|
||||||
the full third-party media is never fetched. Live run `...-20260916091458` on the test
|
the full third-party media is never fetched. Live run `...-20260916091458` on the test
|
||||||
channel: 6 files downloaded, 11 previews cached (YouTube, Reddit, Tenor, TikTok, X). Also
|
channel: 6 files downloaded, 11 previews cached (YouTube, Reddit, Tenor, TikTok, X). Also
|
||||||
|
|||||||
+1
-1
@@ -67,7 +67,7 @@ Event line:
|
|||||||
| `user.search` | `query`, optional `guildId`, optional `limit` | Users from the client cache. Guild members when `guildId` is set. |
|
| `user.search` | `query`, optional `guildId`, optional `limit` | Users from the client cache. Guild members when `guildId` is set. |
|
||||||
| `dm.list` | optional `limit` | Cached DM and group DM channels with recipients, icons, and avatars. Cache-only. |
|
| `dm.list` | optional `limit` | Cached DM and group DM channels with recipients, icons, and avatars. Cache-only. |
|
||||||
| `message.send` | `channelId`, `content` | `{ channelId, messageId, sent: true }` |
|
| `message.send` | `channelId`, `content` | `{ channelId, messageId, sent: true }` |
|
||||||
| `message.read` | `channelId`, `limit` | `{ cached, messages }`. Cache-only. Messages carry the author avatar, attachment metadata (id, filename, URL, content type, size, dimensions, duration), embeds (video, image, and thumbnail URLs), sticker items, forwarded snapshots, a voice-message flag, `editedAt`, and `replyTo`. |
|
| `message.read` | `channelId`, `limit` | `{ cached, messages }`. Cache-only. Messages carry the author avatar, attachment metadata (id, filename, URL, content type, size, dimensions, duration), embeds (video, image, and thumbnail URLs plus their dimensions), sticker items, forwarded snapshots, a voice-message flag, `editedAt`, and `replyTo`. |
|
||||||
| `message.fetch` | `channelId`, `limit`, optional `before` or `after` | `{ fetched, messages }`. Fetches from Discord. Same message shape. |
|
| `message.fetch` | `channelId`, `limit`, optional `before` or `after` | `{ fetched, messages }`. Fetches from Discord. Same message shape. |
|
||||||
| `reaction.add` | `channelId`, `messageId`, `emoji` | `{ added: true }` |
|
| `reaction.add` | `channelId`, `messageId`, `emoji` | `{ added: true }` |
|
||||||
| `dm.send` | `userId`, `content` | `{ userId, channelId, sent: true }` |
|
| `dm.send` | `userId`, `content` | `{ userId, channelId, sent: true }` |
|
||||||
|
|||||||
@@ -175,6 +175,7 @@ Verified live on 2026-09-13: the web UI started the client under auto-Xvfb, the
|
|||||||
| `VESKSLOP_MEDIA_DB_DIR` | `<repo>/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. |
|
| `VESKSLOP_MEDIA_DB_DIR` | `<repo>/media-scrape/db` | Media archive root: per-scrape databases, downloads, previews. |
|
||||||
| `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. |
|
| `VESKSLOP_MEDIA_MAX_FILE_BYTES` | `52428800` | Download size cap (50 MiB). Larger files are skipped. |
|
||||||
| `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. |
|
| `VESKSLOP_MEDIA_PREVIEW_MAX_BYTES` | `5242880` | Preview size cap (5 MiB). Larger previews are skipped. |
|
||||||
|
| `VESKSLOP_FFPROBE` | `ffprobe` | Probe binary used to fill missing dimensions after a download. |
|
||||||
| `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. |
|
| `VESKSLOP_MEDIA_DOWNLOAD_MIN_MS` / `_MAX_MS` | `1000` / `3000` | Gap between downloaded files. |
|
||||||
| `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. |
|
| `VESKSLOP_AUDIO_DUCK` | `30` | Sink volume percent while others speak. |
|
||||||
| `VESKSLOP_TTS_LEAD_MS` | `150` | Lead-in silence before each TTS line (0 disables, max 2000). |
|
| `VESKSLOP_TTS_LEAD_MS` | `150` | Lead-in silence before each TTS line (0 disables, max 2000). |
|
||||||
|
|||||||
+4
-2
@@ -23,8 +23,10 @@ interface MediaService {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`MediaQuery` (implemented): `kind`, `source`, `scrapeId`, `channelId`, `limit`, `offset`. Planned:
|
`MediaQuery` (implemented): `kind`, `source`, `scrapeId`, `channelId`, `authorId`, `aspect`,
|
||||||
`guildId`, `authorId`, `after`, `before`, `text`.
|
`resolution`, `after`, `before`, `limit`, `offset`. The `aspect` and `resolution` values are the
|
||||||
|
preset keys from `filter-presets.ts`; `after`/`before` are ISO times on `posted_at`. Planned:
|
||||||
|
`guildId`, `text`.
|
||||||
|
|
||||||
`StorageStatus`: `usedBytes`, `limitBytes`, `warnPercent`, `overrunPercent`, `byKind`,
|
`StorageStatus`: `usedBytes`, `limitBytes`, `warnPercent`, `overrunPercent`, `byKind`,
|
||||||
`scrapes: [{ id, bytes, state, createdAt }]`.
|
`scrapes: [{ id, bytes, state, createdAt }]`.
|
||||||
|
|||||||
+10
-6
@@ -81,12 +81,16 @@ One SQLite file per scrape (`index.db`, `node:sqlite`).
|
|||||||
| Table | Columns |
|
| Table | Columns |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `messages` | `id`, `author_id`, `author_name`, `timestamp`, `content` |
|
| `messages` | `id`, `author_id`, `author_name`, `timestamp`, `content` |
|
||||||
| `media` | `id`, `kind`, `url`, `provider`, `file_hash`, `preview_hash`, `content_type`, `size`, `width`, `height`, `duration_ms`, `downloaded`, `created_at` |
|
| `media` | `id`, `kind`, `source`, `author_id`, `author_name`, `posted_at`, `url`, `provider`, `file_hash`, `preview_hash`, `content_type`, `size`, `width`, `height`, `aspect_ratio`, `duration_secs`, `downloaded`, `file_path`, `created_at` |
|
||||||
| `media_sources` | `media_id`, `message_id` |
|
| `media_sources` | `media_id`, `message_id` |
|
||||||
| `downloads` | `media_id`, `state`, `attempts`, `error` |
|
| `downloads` | `media_id`, `state`, `attempts`, `error` |
|
||||||
|
|
||||||
Indexes: `(kind, timestamp)` via `media_sources`, `channel` metadata in `scrape.json`, and
|
Indexes: `kind`, `posted_at`, and `media_sources(message_id)`. The database is an index; the media
|
||||||
`file_hash` for dedupe. The database is an index; the media folder holds bytes.
|
folder holds bytes. Every item carries its poster (`author_id`, `author_name`), the post time
|
||||||
|
(`posted_at`), and its dimensions (`width`, `height`, `aspect_ratio`). Filter presets live in
|
||||||
|
`media-scrape/src/filter-presets.ts`: aspect buckets (vertical, square, 4:3, 16:9, 21:9, other) and
|
||||||
|
resolution buckets by the shortest side (SD, HD, FHD, QHD, 4K+, unknown). A missing resolution is
|
||||||
|
filled with `ffprobe` after a download when `VESKSLOP_FFPROBE` is available.
|
||||||
|
|
||||||
## Downloads and previews
|
## Downloads and previews
|
||||||
|
|
||||||
@@ -105,9 +109,9 @@ Previews run after every walk:
|
|||||||
|
|
||||||
- In both modes, Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`)
|
- In both modes, Discord-hosted image and GIF URLs (`cdn.discordapp.com`, `media.discordapp.net`)
|
||||||
are transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`).
|
are transformed to the CDN resizer (`media.discordapp.net?...width=320&height=320&format=webp`).
|
||||||
- In `download` mode only, embed rows also get their stored `preview_url` (YouTube, X, TikTok
|
- In both modes, embed rows also get their stored `preview_url` (YouTube, X, TikTok thumbnails,
|
||||||
thumbnails, and so on) cached. The embed's full media URL is never fetched; embed rows are
|
and so on) cached. The embed's full media URL is never fetched; embed rows are excluded from
|
||||||
excluded from `pendingDownloads`.
|
`pendingDownloads`.
|
||||||
- One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape.
|
- One attempt per row, no retries; a failure marks `preview_error` and never fails the scrape.
|
||||||
- Stored by hash in `thumbs/<sha256>.webp`; `preview_hash` and `preview_bytes` land on the row, and
|
- Stored by hash in `thumbs/<sha256>.webp`; `preview_hash` and `preview_bytes` land on the row, and
|
||||||
the counter is `previewed`. Previews count toward the storage limit.
|
the counter is `previewed`. Previews count toward the storage limit.
|
||||||
|
|||||||
+16
-10
@@ -81,11 +81,13 @@ media-scrape/db/
|
|||||||
are never download targets. The Media tab prefers the local file when one exists, so the archive
|
are never download targets. The Media tab prefers the local file when one exists, so the archive
|
||||||
survives CDN URL expiry.
|
survives CDN URL expiry.
|
||||||
- Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every
|
- Third-party embeds are **never downloaded as media**: YouTube, Instagram, X, TikTok, and every
|
||||||
other embed stays a link. In `download` mode their thumbnail is cached (below); the video itself
|
other embed stays a link. Its thumbnail is cached (below); the video itself is never fetched.
|
||||||
is never fetched.
|
- Previews are cached small (WebP, 320px) in `thumbs/` after every walk, in both modes: Discord
|
||||||
- Previews are cached small (WebP, 320px) in `thumbs/` after every walk. In both modes this covers
|
hosted images and GIFs, and third-party embed thumbnails (YouTube, X, TikTok, and so on), so the
|
||||||
Discord-hosted images and GIFs; in `download` mode it also covers embed thumbnails, so the grid
|
grid does not depend on expiring CDN links. Previews count toward the storage limit.
|
||||||
does not depend on expiring CDN links. Previews count toward the storage limit.
|
- Every item records who posted it, when, and its dimensions: `author_id`, `author_name`,
|
||||||
|
`posted_at`, `width`, `height`, and `aspect_ratio`. In `download` mode a missing resolution is
|
||||||
|
filled in with `ffprobe` after the file lands.
|
||||||
- Discord attachment URLs are signed and expire in about a day. In `links` mode an expired URL is
|
- Discord attachment URLs are signed and expire in about a day. In `links` mode an expired URL is
|
||||||
refreshed by re-fetching its message; in `download` mode the bytes are caught before expiry.
|
refreshed by re-fetching its message; in `download` mode the bytes are caught before expiry.
|
||||||
|
|
||||||
@@ -127,11 +129,15 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun
|
|||||||
|
|
||||||
## Web UI
|
## Web UI
|
||||||
|
|
||||||
- **Media** tab: filter chips (All, Images, GIFs, Videos, Audio), a source filter (any,
|
- **Media** tab: filter chips (All, Images, GIFs, Videos, Audio), and filters for source (any,
|
||||||
attachments, stickers, embeds), and a source-channel filter built from the scrape list. Date and
|
attachments, stickers, embeds), source channel, poster, aspect ratio (Vertical, 1:1, 4:3, 16:9,
|
||||||
text search are planned. Grid with lazy loading; previews come from the cache, then the local
|
21:9, Other), resolution (SD, HD, FHD, QHD, 4K+, Unknown), and a posted-date range (from/to).
|
||||||
file, then the CDN. A scrape-status strip shows progress with Start and Stop. The view fills the
|
Grid with lazy loading; previews come from the cache, then the local file, then the CDN; each
|
||||||
window below the header and the call strip.
|
tile's tooltip shows the poster and posted time. A scrape strip shows live progress with Stop.
|
||||||
|
**New scrape…** opens a popup with Server and Group chat trees (servers expand to channels; each
|
||||||
|
row has Links and Download), and **History** lists each scrape with its start time, duration,
|
||||||
|
pages, messages, media counts, bytes, files, and previews. The view fills the window below the
|
||||||
|
header and the call strip.
|
||||||
- **Bot** tab: slopbot connection state, the scrape queue, and the storage limit with a usage bar.
|
- **Bot** tab: slopbot connection state, the scrape queue, and the storage limit with a usage bar.
|
||||||
The bar is segmented and colour coded by media kind (images, GIFs, video, audio, other) plus the
|
The bar is segmented and colour coded by media kind (images, GIFs, video, audio, other) plus the
|
||||||
database overhead.
|
database overhead.
|
||||||
|
|||||||
+11
-6
@@ -124,11 +124,15 @@ box live in Chat, not Call, and the connection stats are not chips yet.
|
|||||||
- **Media** and **Bot** are scope buttons beside Guild and DM. They open a full-window view below
|
- **Media** and **Bot** are scope buttons beside Guild and DM. They open a full-window view below
|
||||||
the header and the call strip (the shell is covered; only the header, and the call strip when it
|
the header and the call strip (the shell is covered; only the header, and the call strip when it
|
||||||
is visible, stay on screen). Mobile is out of scope for the media feature.
|
is visible, stay on screen). Mobile is out of scope for the media feature.
|
||||||
- Media lists scraped items with kind chips (All, Images, GIFs, Videos, Audio), a source filter
|
- Media lists scraped items with kind chips (All, Images, GIFs, Videos, Audio) and filters for
|
||||||
(any, attachments, stickers, embeds), a source-channel filter built from the scrape list, a
|
source (any, attachments, stickers, embeds), source channel, poster, aspect ratio, resolution,
|
||||||
channel picker with **Start links** / **Start download**, and the active scrapes with **Stop**.
|
and a posted-date range (from/to). **New scrape…** opens a popup with **Server** and **Group
|
||||||
Images and GIFs render from the cached preview when one exists, then the local file, then the
|
chat** tabs: servers expand into their text channels, each channel (and each group chat) has
|
||||||
Discord CDN URL; videos and audio are link cards.
|
**Links** and **Download** buttons, and group chats always use the puppet. **History** lists
|
||||||
|
each scrape with start time, duration, pages, messages, media counts, bytes, files, and previews.
|
||||||
|
The scrape strip shows live progress with a pulse and **Stop**. Images and GIFs render from the
|
||||||
|
cached preview when one exists, then the local file, then the CDN URL; videos and audio are link
|
||||||
|
cards with the poster and posted time in the tooltip.
|
||||||
- Bot shows the slopbot connection state and the scrape queue.
|
- Bot shows the slopbot connection state and the scrape queue.
|
||||||
- The views are self-contained: `poc/public/media.js` and `media.css`, over the `/api/media/*`
|
- The views are self-contained: `poc/public/media.js` and `media.css`, over the `/api/media/*`
|
||||||
routes in the table below. See [media.md](media.md).
|
routes in the table below. See [media.md](media.md).
|
||||||
@@ -252,11 +256,12 @@ replaceable.
|
|||||||
| POST | `/api/audio/stop` | none | Stop speech and clear the queue |
|
| POST | `/api/audio/stop` | none | Stop speech and clear the queue |
|
||||||
| POST | `/api/audio/capture` | `start` | Start or stop capture |
|
| POST | `/api/audio/capture` | `start` | Start or stop capture |
|
||||||
| GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments |
|
| GET | `/api/transcript` | `limit?`, `sessionId?` | Sessions and captured segments |
|
||||||
| GET | `/api/media` | `kind?`, `source?`, `channelId?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `channelId` filters by source channel. `localUrl`/`thumbUrl` when a file or preview exists |
|
| GET | `/api/media` | `kind?`, `source?`, `channelId?`, `authorId?`, `aspect?`, `resolution?`, `after?`, `before?`, `limit?`, `offset?` | Media items across every scrape, newest first. `source` is `attachment`, `sticker`, or `embed`; `aspect` and `resolution` are the preset keys; `after`/`before` are ISO times on `posted_at`. Items carry `authorName`, `postedAt`, `width`, `height`, `aspectRatio`; `localUrl`/`thumbUrl` when a file or preview exists |
|
||||||
| GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension |
|
| GET | `/api/media/:scrapeId/:itemId/file` | none | The downloaded file, content-typed from its extension |
|
||||||
| GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached WebP preview |
|
| GET | `/api/media/:scrapeId/:itemId/thumb` | none | The cached WebP preview |
|
||||||
| GET | `/api/media/scrapes` | none | Every scrape with state and counters |
|
| GET | `/api/media/scrapes` | none | Every scrape with state and counters |
|
||||||
| GET | `/api/media/status` | none | Slopbot connection state and the scrape list |
|
| GET | `/api/media/status` | none | Slopbot connection state and the scrape list |
|
||||||
|
| GET | `/api/media/authors` | none | Distinct posters across every scrape, with counts |
|
||||||
| GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over |
|
| GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over |
|
||||||
| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `backend?`, `confirm?` | Start a media scrape. `backend` is `auto` (default), `slopbot`, or `puppet`. Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` |
|
| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `backend?`, `confirm?` | Start a media scrape. `backend` is `auto` (default), `slopbot`, or `puppet`. Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` |
|
||||||
| POST | `/api/media/scrape/stop` | `id` | Stop a media scrape |
|
| POST | `/api/media/scrape/stop` | `id` | Stop a media scrape |
|
||||||
|
|||||||
@@ -37,11 +37,22 @@ function mapAttachments(message: any): CoreAttachment[] {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Dimensions of the first embed part that has them: video, image, then thumbnail. */
|
||||||
|
function embedDataSize(data: any) {
|
||||||
|
for (const part of [data?.video, data?.image, data?.thumbnail]) {
|
||||||
|
if (part && Number.isFinite(part.width) && Number.isFinite(part.height)) {
|
||||||
|
return { width: part.width, height: part.height };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { width: null, height: null };
|
||||||
|
}
|
||||||
|
|
||||||
function mapEmbeds(message: any): CoreEmbed[] {
|
function mapEmbeds(message: any): CoreEmbed[] {
|
||||||
const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null);
|
const assetUrl = (value: any) => value?.url ?? (typeof value === "string" ? value : null);
|
||||||
|
|
||||||
return collectionValues(message?.embeds).map((embed: any) => {
|
return collectionValues(message?.embeds).map((embed: any) => {
|
||||||
const data = embed?.data ?? embed ?? {};
|
const data = embed?.data ?? embed ?? {};
|
||||||
|
const size = embedDataSize(data);
|
||||||
return {
|
return {
|
||||||
type: data.type ?? null,
|
type: data.type ?? null,
|
||||||
url: data.url ?? null,
|
url: data.url ?? null,
|
||||||
@@ -49,7 +60,9 @@ function mapEmbeds(message: any): CoreEmbed[] {
|
|||||||
provider: data.provider?.name ?? null,
|
provider: data.provider?.name ?? null,
|
||||||
videoUrl: assetUrl(data.video),
|
videoUrl: assetUrl(data.video),
|
||||||
imageUrl: assetUrl(data.image),
|
imageUrl: assetUrl(data.image),
|
||||||
thumbnailUrl: assetUrl(data.thumbnail)
|
thumbnailUrl: assetUrl(data.thumbnail),
|
||||||
|
width: size.width,
|
||||||
|
height: size.height
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,11 +3,36 @@
|
|||||||
* Rules: docs/media-scraper.md. Third-party embeds are never targets here.
|
* Rules: docs/media-scraper.md. Third-party embeds are never targets here.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { execFile } from "node:child_process";
|
||||||
import { createHash } from "node:crypto";
|
import { createHash } from "node:crypto";
|
||||||
import { writeFileSync } from "node:fs";
|
import { writeFileSync } from "node:fs";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
|
import { promisify } from "node:util";
|
||||||
import type { MediaRow } from "./store.js";
|
import type { MediaRow } from "./store.js";
|
||||||
|
|
||||||
|
const execFileAsync = promisify(execFile);
|
||||||
|
|
||||||
|
/** Read pixel dimensions with ffprobe. Null when the tool is missing or the file is not media. */
|
||||||
|
export async function probeDimensions(
|
||||||
|
path: string,
|
||||||
|
ffprobe = process.env.VESKSLOP_FFPROBE ?? "ffprobe"
|
||||||
|
): Promise<{ width: number; height: number } | null> {
|
||||||
|
try {
|
||||||
|
const { stdout } = await execFileAsync(
|
||||||
|
ffprobe,
|
||||||
|
["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "json", path],
|
||||||
|
{ timeout: 15_000 }
|
||||||
|
);
|
||||||
|
const stream = JSON.parse(stdout)?.streams?.[0];
|
||||||
|
if (Number.isFinite(stream?.width) && Number.isFinite(stream?.height)) {
|
||||||
|
return { width: stream.width, height: stream.height };
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// missing ffprobe, a timeout, or a non-visual file
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DownloadOptions {
|
export interface DownloadOptions {
|
||||||
fetchImpl?: typeof fetch;
|
fetchImpl?: typeof fetch;
|
||||||
/** Skip files larger than this. Default 50 MiB. */
|
/** Skip files larger than this. Default 50 MiB. */
|
||||||
|
|||||||
@@ -4,7 +4,13 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { downloadGapMs, downloadOne, downloadPreview, type DownloadOptions } from "./download.js";
|
import {
|
||||||
|
downloadGapMs,
|
||||||
|
downloadOne,
|
||||||
|
downloadPreview,
|
||||||
|
probeDimensions,
|
||||||
|
type DownloadOptions
|
||||||
|
} from "./download.js";
|
||||||
import {
|
import {
|
||||||
createScrapeDirs,
|
createScrapeDirs,
|
||||||
listScrapes,
|
listScrapes,
|
||||||
@@ -50,6 +56,8 @@ export interface ScrapeEngineOptions {
|
|||||||
download?: DownloadOptions;
|
download?: DownloadOptions;
|
||||||
/** Cache Discord-hosted image previews after every walk. Default true. */
|
/** Cache Discord-hosted image previews after every walk. Default true. */
|
||||||
previews?: boolean;
|
previews?: boolean;
|
||||||
|
/** Read dimensions from a downloaded file. Defaults to ffprobe. */
|
||||||
|
probe?: (path: string) => Promise<{ width: number; height: number } | null>;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface QueuedJob {
|
interface QueuedJob {
|
||||||
@@ -244,6 +252,9 @@ export class ScrapeEngine {
|
|||||||
meta.lastError = error instanceof Error ? error.message : String(error);
|
meta.lastError = error instanceof Error ? error.message : String(error);
|
||||||
this.log(`${meta.id}: failed: ${meta.lastError}`);
|
this.log(`${meta.id}: failed: ${meta.lastError}`);
|
||||||
} finally {
|
} finally {
|
||||||
|
if ((meta.state === "done" || meta.state === "failed") && !meta.finishedAt) {
|
||||||
|
meta.finishedAt = new Date(this.now()).toISOString();
|
||||||
|
}
|
||||||
this.persist(dir, meta);
|
this.persist(dir, meta);
|
||||||
store.close();
|
store.close();
|
||||||
this.active.set(meta.id, meta);
|
this.active.set(meta.id, meta);
|
||||||
@@ -273,6 +284,11 @@ export class ScrapeEngine {
|
|||||||
bytes: outcome.bytes ?? 0
|
bytes: outcome.bytes ?? 0
|
||||||
});
|
});
|
||||||
meta.downloaded = (meta.downloaded ?? 0) + 1;
|
meta.downloaded = (meta.downloaded ?? 0) + 1;
|
||||||
|
|
||||||
|
if (row.width === null || row.height === null) {
|
||||||
|
const dimensions = await (this.options.probe ?? probeDimensions)(join(dir, outcome.path));
|
||||||
|
if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
store.markDownloadError(row.id, outcome.error ?? outcome.state);
|
store.markDownloadError(row.id, outcome.error ?? outcome.state);
|
||||||
if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1;
|
if (outcome.state === "failed") meta.downloadErrors = (meta.downloadErrors ?? 0) + 1;
|
||||||
@@ -283,11 +299,10 @@ export class ScrapeEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Cache small previews. In download mode this includes embed thumbnails. */
|
/** Cache small previews. Embed thumbnails are cached in both modes. */
|
||||||
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
|
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
|
||||||
const embedPreviews = meta.mode === "download";
|
const options = { ...(this.options.download ?? {}), embedPreviews: true };
|
||||||
const options = { ...(this.options.download ?? {}), embedPreviews };
|
const pending = store.pendingPreviews(5000, true);
|
||||||
const pending = store.pendingPreviews(5000, embedPreviews);
|
|
||||||
|
|
||||||
for (const row of pending) {
|
for (const row of pending) {
|
||||||
if (this.stopRequested.has(meta.id)) {
|
if (this.stopRequested.has(meta.id)) {
|
||||||
|
|||||||
@@ -76,8 +76,8 @@ function extractEmbed(embed: CoreEmbed): ExtractedMedia | null {
|
|||||||
provider,
|
provider,
|
||||||
downloaded: false,
|
downloaded: false,
|
||||||
size: null,
|
size: null,
|
||||||
width: null,
|
width: embed.width ?? null,
|
||||||
height: null,
|
height: embed.height ?? null,
|
||||||
durationSecs: null
|
durationSecs: null
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
/*
|
||||||
|
* Query presets for the media filters: aspect ratio and resolution buckets.
|
||||||
|
* Pure; the SQL fragments are fixed strings, never built from user input.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const ASPECT_PRESETS = ["vertical", "square", "wide43", "wide169", "ultrawide", "other"] as const;
|
||||||
|
export type AspectPreset = (typeof ASPECT_PRESETS)[number];
|
||||||
|
|
||||||
|
/** Aspect-ratio buckets: vertical/shorts, square, 4:3, 16:9, 21:9, everything else. */
|
||||||
|
export function aspectCondition(preset: string): string | null {
|
||||||
|
switch (preset) {
|
||||||
|
case "vertical":
|
||||||
|
return "aspect_ratio < 0.95";
|
||||||
|
case "square":
|
||||||
|
return "aspect_ratio >= 0.95 AND aspect_ratio < 1.05";
|
||||||
|
case "wide43":
|
||||||
|
return "aspect_ratio >= 1.05 AND aspect_ratio < 1.55";
|
||||||
|
case "wide169":
|
||||||
|
return "aspect_ratio >= 1.55 AND aspect_ratio < 2.05";
|
||||||
|
case "ultrawide":
|
||||||
|
return "aspect_ratio >= 2.05 AND aspect_ratio < 2.8";
|
||||||
|
case "other":
|
||||||
|
return "(aspect_ratio IS NULL OR aspect_ratio >= 2.8)";
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const RESOLUTION_PRESETS = ["sd", "hd", "fhd", "qhd", "uhd", "unknown"] as const;
|
||||||
|
export type ResolutionPreset = (typeof RESOLUTION_PRESETS)[number];
|
||||||
|
|
||||||
|
/** Resolution buckets by the shortest side, so vertical 1080x1920 counts as FHD. */
|
||||||
|
export function resolutionCondition(preset: string): string | null {
|
||||||
|
switch (preset) {
|
||||||
|
case "sd":
|
||||||
|
return "MIN(width, height) <= 480";
|
||||||
|
case "hd":
|
||||||
|
return "MIN(width, height) > 480 AND MIN(width, height) <= 900";
|
||||||
|
case "fhd":
|
||||||
|
return "MIN(width, height) > 900 AND MIN(width, height) <= 1260";
|
||||||
|
case "qhd":
|
||||||
|
return "MIN(width, height) > 1260 AND MIN(width, height) <= 1800";
|
||||||
|
case "uhd":
|
||||||
|
return "MIN(width, height) > 1800";
|
||||||
|
case "unknown":
|
||||||
|
return "(width IS NULL OR height IS NULL)";
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,14 @@
|
|||||||
|
|
||||||
export { extractMedia, type ExtractedMedia } from "./extract.js";
|
export { extractMedia, type ExtractedMedia } from "./extract.js";
|
||||||
export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
|
export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
|
||||||
|
export {
|
||||||
|
ASPECT_PRESETS,
|
||||||
|
RESOLUTION_PRESETS,
|
||||||
|
aspectCondition,
|
||||||
|
resolutionCondition,
|
||||||
|
type AspectPreset,
|
||||||
|
type ResolutionPreset
|
||||||
|
} from "./filter-presets.js";
|
||||||
export {
|
export {
|
||||||
downloadGapMs,
|
downloadGapMs,
|
||||||
downloadOne,
|
downloadOne,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import { join } from "node:path";
|
|||||||
import { DatabaseSync } from "node:sqlite";
|
import { DatabaseSync } from "node:sqlite";
|
||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
import { extractMedia } from "./extract.js";
|
import { extractMedia } from "./extract.js";
|
||||||
|
import { aspectCondition, resolutionCondition } from "./filter-presets.js";
|
||||||
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
|
import { emptyMediaCounts, type CoreMessage, type MediaKind, type ScrapeMeta } from "./types.js";
|
||||||
|
|
||||||
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
|
/** Default DB root: `media-scrape/db`, beside the module. Override with VESKSLOP_MEDIA_DB_DIR. */
|
||||||
@@ -77,6 +78,12 @@ export function dirSizeBytes(dir: string): number {
|
|||||||
return total;
|
return total;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Width / height, rounded to three decimals. Null when either side is missing or zero. */
|
||||||
|
export function aspectRatioOf(width: number | null | undefined, height: number | null | undefined): number | null {
|
||||||
|
if (!Number.isFinite(width) || !Number.isFinite(height) || !width || !height) return null;
|
||||||
|
return Math.round(((width as number) / (height as number)) * 1000) / 1000;
|
||||||
|
}
|
||||||
|
|
||||||
export function createScrapeDirs(root: string, id: string): string {
|
export function createScrapeDirs(root: string, id: string): string {
|
||||||
const dir = scrapeDir(root, id);
|
const dir = scrapeDir(root, id);
|
||||||
mkdirSync(join(dir, "media"), { recursive: true });
|
mkdirSync(join(dir, "media"), { recursive: true });
|
||||||
@@ -99,12 +106,16 @@ export interface MediaRow {
|
|||||||
id: number;
|
id: number;
|
||||||
kind: MediaKind;
|
kind: MediaKind;
|
||||||
source: string | null;
|
source: string | null;
|
||||||
|
author_id: string | null;
|
||||||
|
author_name: string | null;
|
||||||
|
posted_at: string | null;
|
||||||
url: string;
|
url: string;
|
||||||
provider: string | null;
|
provider: string | null;
|
||||||
content_type: string | null;
|
content_type: string | null;
|
||||||
size: number | null;
|
size: number | null;
|
||||||
width: number | null;
|
width: number | null;
|
||||||
height: number | null;
|
height: number | null;
|
||||||
|
aspect_ratio: number | null;
|
||||||
duration_secs: number | null;
|
duration_secs: number | null;
|
||||||
downloaded: number;
|
downloaded: number;
|
||||||
preview_url: string | null;
|
preview_url: string | null;
|
||||||
@@ -131,12 +142,16 @@ CREATE TABLE IF NOT EXISTS media (
|
|||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
kind TEXT NOT NULL,
|
kind TEXT NOT NULL,
|
||||||
source TEXT,
|
source TEXT,
|
||||||
|
author_id TEXT,
|
||||||
|
author_name TEXT,
|
||||||
|
posted_at TEXT,
|
||||||
url TEXT NOT NULL UNIQUE,
|
url TEXT NOT NULL UNIQUE,
|
||||||
provider TEXT,
|
provider TEXT,
|
||||||
content_type TEXT,
|
content_type TEXT,
|
||||||
size INTEGER,
|
size INTEGER,
|
||||||
width INTEGER,
|
width INTEGER,
|
||||||
height INTEGER,
|
height INTEGER,
|
||||||
|
aspect_ratio REAL,
|
||||||
duration_secs REAL,
|
duration_secs REAL,
|
||||||
downloaded INTEGER NOT NULL DEFAULT 0,
|
downloaded INTEGER NOT NULL DEFAULT 0,
|
||||||
preview_url TEXT,
|
preview_url TEXT,
|
||||||
@@ -157,6 +172,7 @@ CREATE TABLE IF NOT EXISTS media_sources (
|
|||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
|
CREATE INDEX IF NOT EXISTS idx_media_kind ON media(kind);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_media_posted ON media(posted_at);
|
||||||
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
|
CREATE INDEX IF NOT EXISTS idx_media_sources_message ON media_sources(message_id);
|
||||||
`;
|
`;
|
||||||
|
|
||||||
@@ -178,9 +194,13 @@ export class ScrapeStore {
|
|||||||
if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
|
if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
|
||||||
if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
|
if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
|
||||||
if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
|
if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
|
||||||
|
if (!names.has("author_id")) this.db.exec("ALTER TABLE media ADD COLUMN author_id TEXT");
|
||||||
|
if (!names.has("author_name")) this.db.exec("ALTER TABLE media ADD COLUMN author_name TEXT");
|
||||||
|
if (!names.has("posted_at")) this.db.exec("ALTER TABLE media ADD COLUMN posted_at TEXT");
|
||||||
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
|
if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
|
||||||
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
|
if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
|
||||||
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
|
if (!names.has("preview_error")) this.db.exec("ALTER TABLE media ADD COLUMN preview_error TEXT");
|
||||||
|
if (!names.has("aspect_ratio")) this.db.exec("ALTER TABLE media ADD COLUMN aspect_ratio REAL");
|
||||||
}
|
}
|
||||||
|
|
||||||
close(): void {
|
close(): void {
|
||||||
@@ -194,7 +214,7 @@ export class ScrapeStore {
|
|||||||
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
|
"INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
|
||||||
);
|
);
|
||||||
const insertMedia = this.db.prepare(
|
const insertMedia = this.db.prepare(
|
||||||
"INSERT OR IGNORE INTO media (kind, source, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
"INSERT OR IGNORE INTO media (kind, source, author_id, author_name, posted_at, url, provider, content_type, size, width, height, aspect_ratio, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||||||
);
|
);
|
||||||
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
|
const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
|
||||||
const linkSource = this.db.prepare(
|
const linkSource = this.db.prepare(
|
||||||
@@ -217,12 +237,16 @@ export class ScrapeStore {
|
|||||||
const result = insertMedia.run(
|
const result = insertMedia.run(
|
||||||
item.kind,
|
item.kind,
|
||||||
item.source,
|
item.source,
|
||||||
|
message.author.id,
|
||||||
|
message.author.username,
|
||||||
|
message.timestamp,
|
||||||
item.url,
|
item.url,
|
||||||
item.provider,
|
item.provider,
|
||||||
item.contentType,
|
item.contentType,
|
||||||
item.size,
|
item.size,
|
||||||
item.width,
|
item.width,
|
||||||
item.height,
|
item.height,
|
||||||
|
aspectRatioOf(item.width, item.height),
|
||||||
item.durationSecs,
|
item.durationSecs,
|
||||||
item.downloaded ? 1 : 0,
|
item.downloaded ? 1 : 0,
|
||||||
item.previewUrl,
|
item.previewUrl,
|
||||||
@@ -321,6 +345,23 @@ export class ScrapeStore {
|
|||||||
this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
|
this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Fill in dimensions after a download probe. */
|
||||||
|
updateDimensions(id: number, width: number, height: number): void {
|
||||||
|
this.db
|
||||||
|
.prepare("UPDATE media SET width = ?, height = ?, aspect_ratio = ? WHERE id = ?")
|
||||||
|
.run(width, height, aspectRatioOf(width, height), id);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Distinct posters with their item counts, biggest first. */
|
||||||
|
authors(limit = 200): Array<{ id: string | null; name: string | null; count: number }> {
|
||||||
|
const rows = this.db
|
||||||
|
.prepare(
|
||||||
|
"SELECT author_id AS id, author_name AS name, COUNT(*) AS count FROM media WHERE author_name IS NOT NULL GROUP BY author_id, author_name ORDER BY count DESC LIMIT ?"
|
||||||
|
)
|
||||||
|
.all(limit) as unknown as Array<{ id: string | null; name: string | null; count: number }>;
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
getMedia(id: number): MediaRow | null {
|
getMedia(id: number): MediaRow | null {
|
||||||
const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
|
const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
|
||||||
| MediaRow
|
| MediaRow
|
||||||
@@ -329,13 +370,24 @@ export class ScrapeStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
queryMedia(
|
queryMedia(
|
||||||
options: { kind?: MediaKind; source?: string; limit?: number; offset?: number } = {}
|
options: {
|
||||||
|
kind?: MediaKind;
|
||||||
|
source?: string;
|
||||||
|
authorId?: string;
|
||||||
|
aspect?: string;
|
||||||
|
resolution?: string;
|
||||||
|
after?: string;
|
||||||
|
before?: string;
|
||||||
|
limit?: number;
|
||||||
|
offset?: number;
|
||||||
|
} = {}
|
||||||
): MediaRow[] {
|
): MediaRow[] {
|
||||||
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
|
const limit = Math.max(1, Math.min(options.limit ?? 50, 500));
|
||||||
const offset = Math.max(0, options.offset ?? 0);
|
const offset = Math.max(0, options.offset ?? 0);
|
||||||
|
|
||||||
const conditions: string[] = [];
|
const conditions: string[] = [];
|
||||||
const values: Array<string | number> = [];
|
const values: Array<string | number> = [];
|
||||||
|
|
||||||
if (options.kind) {
|
if (options.kind) {
|
||||||
conditions.push("kind = ?");
|
conditions.push("kind = ?");
|
||||||
values.push(options.kind);
|
values.push(options.kind);
|
||||||
@@ -344,6 +396,24 @@ export class ScrapeStore {
|
|||||||
conditions.push("source = ?");
|
conditions.push("source = ?");
|
||||||
values.push(options.source);
|
values.push(options.source);
|
||||||
}
|
}
|
||||||
|
if (options.authorId) {
|
||||||
|
conditions.push("author_id = ?");
|
||||||
|
values.push(options.authorId);
|
||||||
|
}
|
||||||
|
if (options.after) {
|
||||||
|
conditions.push("posted_at >= ?");
|
||||||
|
values.push(options.after);
|
||||||
|
}
|
||||||
|
if (options.before) {
|
||||||
|
conditions.push("posted_at <= ?");
|
||||||
|
values.push(options.before);
|
||||||
|
}
|
||||||
|
|
||||||
|
const aspect = options.aspect ? aspectCondition(options.aspect) : null;
|
||||||
|
if (aspect) conditions.push(aspect);
|
||||||
|
|
||||||
|
const resolution = options.resolution ? resolutionCondition(options.resolution) : null;
|
||||||
|
if (resolution) conditions.push(resolution);
|
||||||
|
|
||||||
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
||||||
const rows = this.db
|
const rows = this.db
|
||||||
|
|||||||
@@ -39,6 +39,9 @@ export interface CoreEmbed {
|
|||||||
videoUrl: string | null;
|
videoUrl: string | null;
|
||||||
imageUrl: string | null;
|
imageUrl: string | null;
|
||||||
thumbnailUrl: string | null;
|
thumbnailUrl: string | null;
|
||||||
|
/** Dimensions of the video, image, or thumbnail, first one Discord provides. */
|
||||||
|
width?: number | null;
|
||||||
|
height?: number | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CoreSticker {
|
export interface CoreSticker {
|
||||||
@@ -89,6 +92,8 @@ export interface ScrapeMeta {
|
|||||||
downloadErrors?: number;
|
downloadErrors?: number;
|
||||||
/** Cached image previews. */
|
/** Cached image previews. */
|
||||||
previewed?: number;
|
previewed?: number;
|
||||||
|
/** When the scrape reached done or failed. */
|
||||||
|
finishedAt?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function emptyMediaCounts(): Record<MediaKind, number> {
|
export function emptyMediaCounts(): Record<MediaKind, number> {
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import { downloadOne, downloadPreview, previewUrlFor } from "../src/download.js"
|
|||||||
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
|
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
|
||||||
import { extractMedia } from "../src/extract.js";
|
import { extractMedia } from "../src/extract.js";
|
||||||
import { PuppetBackend } from "../src/puppet-backend.js";
|
import { PuppetBackend } from "../src/puppet-backend.js";
|
||||||
|
import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
|
||||||
import { isNearLimit, planRetention } from "../src/retention.js";
|
import { isNearLimit, planRetention } from "../src/retention.js";
|
||||||
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
|
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
|
||||||
import type { CoreAttachment, CoreMessage } from "../src/types.js";
|
import type { CoreAttachment, CoreMessage } from "../src/types.js";
|
||||||
@@ -196,6 +197,7 @@ void (async () => {
|
|||||||
assert.equal(meta?.messages, 3);
|
assert.equal(meta?.messages, 3);
|
||||||
assert.equal(meta?.cursor, "99");
|
assert.equal(meta?.cursor, "99");
|
||||||
assert.equal(meta?.media.image, 1);
|
assert.equal(meta?.media.image, 1);
|
||||||
|
assert.ok(meta?.finishedAt);
|
||||||
assert.equal(backend.calls[0].limit, 2);
|
assert.equal(backend.calls[0].limit, 2);
|
||||||
assert.equal(backend.calls[1].before, "100");
|
assert.equal(backend.calls[1].before, "100");
|
||||||
|
|
||||||
@@ -209,6 +211,9 @@ void (async () => {
|
|||||||
assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
|
assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
|
||||||
assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
|
assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
|
||||||
assert.equal(stored.storedBytesByKind().image, 0);
|
assert.equal(stored.storedBytesByKind().image, 0);
|
||||||
|
const row = stored.queryMedia({ kind: "image" })[0];
|
||||||
|
assert.equal(row.author_name, "tester");
|
||||||
|
assert.equal(row.posted_at, "2026-09-16T00:00:00.000Z");
|
||||||
stored.close();
|
stored.close();
|
||||||
|
|
||||||
assert.equal(listScrapes(root).length, 1);
|
assert.equal(listScrapes(root).length, 1);
|
||||||
@@ -405,14 +410,30 @@ void (async () => {
|
|||||||
assert.equal(existsSync(join(root, started.id, row.file_path)), true);
|
assert.equal(existsSync(join(root, started.id, row.file_path)), true);
|
||||||
});
|
});
|
||||||
|
|
||||||
await test("store filters by source and kind", () => {
|
await test("aspect and resolution presets have fixed buckets", () => {
|
||||||
|
assert.equal(aspectCondition("vertical"), "aspect_ratio < 0.95");
|
||||||
|
assert.equal(aspectCondition("wide169"), "aspect_ratio >= 1.55 AND aspect_ratio < 2.05");
|
||||||
|
assert.equal(aspectCondition("nope"), null);
|
||||||
|
assert.equal(resolutionCondition("fhd"), "MIN(width, height) > 900 AND MIN(width, height) <= 1260");
|
||||||
|
assert.equal(resolutionCondition("unknown"), "(width IS NULL OR height IS NULL)");
|
||||||
|
assert.equal(resolutionCondition("nope"), null);
|
||||||
|
});
|
||||||
|
|
||||||
|
await test("store filters by source, kind, author, ratio, resolution, and dates", () => {
|
||||||
const dir = join(root, "store-source");
|
const dir = join(root, "store-source");
|
||||||
mkdirSync(dir, { recursive: true });
|
mkdirSync(dir, { recursive: true });
|
||||||
|
|
||||||
const store = new ScrapeStore(dir);
|
const store = new ScrapeStore(dir);
|
||||||
store.insertPage([
|
store.insertPage([
|
||||||
message("700", {
|
message("700", {
|
||||||
attachments: [attachment({ filename: "pic.png", contentType: "image/png" })],
|
attachments: [
|
||||||
|
attachment({
|
||||||
|
filename: "pic.png",
|
||||||
|
contentType: "image/png",
|
||||||
|
width: 640,
|
||||||
|
height: 360
|
||||||
|
})
|
||||||
|
],
|
||||||
embeds: [
|
embeds: [
|
||||||
{
|
{
|
||||||
type: "video",
|
type: "video",
|
||||||
@@ -432,6 +453,59 @@ void (async () => {
|
|||||||
assert.equal(store.queryMedia({ source: "sticker" }).length, 0);
|
assert.equal(store.queryMedia({ source: "sticker" }).length, 0);
|
||||||
assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1);
|
assert.equal(store.queryMedia({ kind: "image", source: "attachment" }).length, 1);
|
||||||
assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0);
|
assert.equal(store.queryMedia({ kind: "video", source: "attachment" }).length, 0);
|
||||||
|
|
||||||
|
assert.equal(store.queryMedia({ aspect: "wide169" }).length, 1);
|
||||||
|
assert.equal(store.queryMedia({ aspect: "vertical" }).length, 0);
|
||||||
|
assert.equal(store.queryMedia({ resolution: "sd" }).length, 1);
|
||||||
|
assert.equal(store.queryMedia({ resolution: "fhd" }).length, 0);
|
||||||
|
assert.equal(store.queryMedia({ authorId: "1" }).length, 2);
|
||||||
|
assert.equal(store.queryMedia({ authorId: "999" }).length, 0);
|
||||||
|
assert.equal(store.queryMedia({ after: "2026-09-16T00:00:00.000Z" }).length, 2);
|
||||||
|
assert.equal(store.queryMedia({ before: "2026-09-15T00:00:00.000Z" }).length, 0);
|
||||||
|
|
||||||
|
const [wide] = store.queryMedia({ kind: "image" });
|
||||||
|
assert.equal(wide.aspect_ratio, 1.778);
|
||||||
|
store.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
await test("engine probes dimensions when the metadata lacks them", async () => {
|
||||||
|
const backend = new FakeBackend([
|
||||||
|
{
|
||||||
|
messages: [
|
||||||
|
message("701", {
|
||||||
|
attachments: [attachment({ filename: "probe.png", contentType: "image/png" })]
|
||||||
|
})
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]);
|
||||||
|
|
||||||
|
const engine = new ScrapeEngine({
|
||||||
|
rootDir: root,
|
||||||
|
pageSize: 2,
|
||||||
|
pageMinMs: 0,
|
||||||
|
pageMaxMs: 0,
|
||||||
|
sleep: async () => {},
|
||||||
|
random: () => 0,
|
||||||
|
log: () => {},
|
||||||
|
previews: false,
|
||||||
|
probe: async () => ({ width: 1080, height: 1920 }),
|
||||||
|
download: {
|
||||||
|
fetchImpl: async () =>
|
||||||
|
new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
|
||||||
|
minGapMs: 0,
|
||||||
|
maxGapMs: 0,
|
||||||
|
sleep: async () => {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const started = await engine.start({ channelId: "129", guildId: null, mode: "download", backend });
|
||||||
|
await engine.processQueue();
|
||||||
|
|
||||||
|
const store = new ScrapeStore(join(root, started.id));
|
||||||
|
const row = store.queryMedia({ kind: "image" })[0];
|
||||||
|
assert.equal(row.width, 1080);
|
||||||
|
assert.equal(row.height, 1920);
|
||||||
|
assert.equal(row.aspect_ratio, 0.563);
|
||||||
store.close();
|
store.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -502,7 +576,7 @@ void (async () => {
|
|||||||
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
|
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
|
||||||
});
|
});
|
||||||
|
|
||||||
await test("engine caches previews in links mode and skips embeds", async () => {
|
await test("engine caches previews in links mode, including embed thumbnails", async () => {
|
||||||
const backend = new FakeBackend([
|
const backend = new FakeBackend([
|
||||||
{
|
{
|
||||||
messages: [
|
messages: [
|
||||||
@@ -551,8 +625,9 @@ void (async () => {
|
|||||||
|
|
||||||
const meta = engine.get(started.id);
|
const meta = engine.get(started.id);
|
||||||
assert.equal(meta?.state, "done");
|
assert.equal(meta?.state, "done");
|
||||||
assert.equal(meta?.previewed, 1);
|
assert.equal(meta?.previewed, 2);
|
||||||
assert.equal(fetched.some(url => url.includes("ytimg.com")), false);
|
assert.equal(fetched.some(url => url.includes("ytimg.com")), true);
|
||||||
|
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
|
||||||
|
|
||||||
const store = new ScrapeStore(join(root, started.id));
|
const store = new ScrapeStore(join(root, started.id));
|
||||||
const row = store.queryMedia({ kind: "image" })[0];
|
const row = store.queryMedia({ kind: "image" })[0];
|
||||||
|
|||||||
+5
-1
@@ -30,7 +30,11 @@ async function request(path, options) {
|
|||||||
} catch {
|
} catch {
|
||||||
throw new Error(`request failed (${response.status})`);
|
throw new Error(`request failed (${response.status})`);
|
||||||
}
|
}
|
||||||
if (!body.ok) throw new Error(body.error ?? `request failed (${response.status})`);
|
if (!body.ok) {
|
||||||
|
const error = new Error(body.error ?? `request failed (${response.status})`);
|
||||||
|
if (body.code) error.code = body.code;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
return body.result;
|
return body.result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -202,9 +202,87 @@
|
|||||||
box-shadow: inset 0 0 0 1px var(--color-neutral-800);
|
box-shadow: inset 0 0 0 1px var(--color-neutral-800);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.vk-media-modal-box.vk-media-wide {
|
||||||
|
width: min(760px, 94vw);
|
||||||
|
height: min(80vh, 720px);
|
||||||
|
}
|
||||||
|
|
||||||
.vk-media-modal-list {
|
.vk-media-modal-list {
|
||||||
max-height: 220px;
|
max-height: 220px;
|
||||||
overflow-y: auto;
|
overflow-y: auto;
|
||||||
font: 400 11px/1.5 ui-monospace, Menlo, monospace;
|
font: 400 11px/1.5 ui-monospace, Menlo, monospace;
|
||||||
color: var(--color-neutral-400);
|
color: var(--color-neutral-400);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.vk-media-wide .vk-media-modal-list {
|
||||||
|
max-height: none;
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vk-media-history-row {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 2px;
|
||||||
|
padding: 7px 0;
|
||||||
|
border-bottom: 1px solid var(--color-neutral-900);
|
||||||
|
}
|
||||||
|
|
||||||
|
.vk-media-tree {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 6px;
|
||||||
|
flex: 1;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vk-media-tree-head {
|
||||||
|
padding: 6px 8px;
|
||||||
|
border: none;
|
||||||
|
border-radius: var(--radius-sm, 6px);
|
||||||
|
background: var(--color-neutral-900);
|
||||||
|
color: var(--color-neutral-200);
|
||||||
|
font: 500 12px/1.2 inherit;
|
||||||
|
text-align: left;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vk-media-tree-children {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 3px;
|
||||||
|
margin: 4px 0 4px 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vk-media-channel {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vk-media-channel-name {
|
||||||
|
flex: 1;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
font: 400 11px/1.4 ui-monospace, Menlo, monospace;
|
||||||
|
color: var(--color-neutral-300);
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes vk-media-pulse {
|
||||||
|
0%,
|
||||||
|
100% {
|
||||||
|
opacity: 0.25;
|
||||||
|
}
|
||||||
|
50% {
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
.vk-media-pulse {
|
||||||
|
flex: none;
|
||||||
|
width: 8px;
|
||||||
|
height: 8px;
|
||||||
|
border-radius: 50%;
|
||||||
|
background: var(--color-accent-400, #4f8cff);
|
||||||
|
animation: vk-media-pulse 1.2s ease-in-out infinite;
|
||||||
|
}
|
||||||
|
|||||||
+327
-28
@@ -18,8 +18,19 @@
|
|||||||
items: [],
|
items: [],
|
||||||
scrapes: [],
|
scrapes: [],
|
||||||
channels: [],
|
channels: [],
|
||||||
|
guilds: [],
|
||||||
|
dms: [],
|
||||||
loadedChannels: false,
|
loadedChannels: false,
|
||||||
filters: { source: "", channelId: "" }
|
authors: null,
|
||||||
|
filters: {
|
||||||
|
source: "",
|
||||||
|
channelId: "",
|
||||||
|
authorId: "",
|
||||||
|
aspect: "",
|
||||||
|
resolution: "",
|
||||||
|
after: "",
|
||||||
|
before: ""
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const overlay = document.createElement("div");
|
const overlay = document.createElement("div");
|
||||||
@@ -95,8 +106,9 @@
|
|||||||
if (state.loadedChannels) return;
|
if (state.loadedChannels) return;
|
||||||
try {
|
try {
|
||||||
const data = await window.VK.api.get("/api/state");
|
const data = await window.VK.api.get("/api/state");
|
||||||
|
state.guilds = data?.guilds ?? [];
|
||||||
state.channels = [];
|
state.channels = [];
|
||||||
for (const guild of data?.guilds ?? []) {
|
for (const guild of state.guilds) {
|
||||||
for (const channel of guild.channels ?? []) {
|
for (const channel of guild.channels ?? []) {
|
||||||
if (channel.type !== 0 && channel.type !== 5) continue;
|
if (channel.type !== 0 && channel.type !== 5) continue;
|
||||||
state.channels.push({
|
state.channels.push({
|
||||||
@@ -109,8 +121,9 @@
|
|||||||
|
|
||||||
// DMs and group DMs: the bot cannot see them, so they always use the puppet.
|
// DMs and group DMs: the bot cannot see them, so they always use the puppet.
|
||||||
try {
|
try {
|
||||||
const dms = await window.VK.api.get("/api/dms?limit=50");
|
const dmData = await window.VK.api.get("/api/dms?limit=50");
|
||||||
for (const dm of dms ?? []) {
|
state.dms = dmData?.channels ?? [];
|
||||||
|
for (const dm of state.dms) {
|
||||||
const names = (dm.recipients ?? [])
|
const names = (dm.recipients ?? [])
|
||||||
.map(recipient => recipient.username)
|
.map(recipient => recipient.username)
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
@@ -122,15 +135,25 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// no cached DMs yet
|
state.dms = [];
|
||||||
}
|
}
|
||||||
|
|
||||||
state.loadedChannels = true;
|
state.loadedChannels = true;
|
||||||
} catch {
|
} catch {
|
||||||
|
state.guilds = [];
|
||||||
state.channels = [];
|
state.channels = [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadAuthors() {
|
||||||
|
if (state.authors) return;
|
||||||
|
try {
|
||||||
|
state.authors = await window.VK.api.get("/api/media/authors");
|
||||||
|
} catch {
|
||||||
|
state.authors = [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function loadScrapes() {
|
async function loadScrapes() {
|
||||||
try {
|
try {
|
||||||
state.scrapes = await window.VK.api.get("/api/media/scrapes");
|
state.scrapes = await window.VK.api.get("/api/media/scrapes");
|
||||||
@@ -151,8 +174,13 @@
|
|||||||
const kind = state.kind === "all" ? "" : `&kind=${state.kind}`;
|
const kind = state.kind === "all" ? "" : `&kind=${state.kind}`;
|
||||||
const source = state.filters.source ? `&source=${state.filters.source}` : "";
|
const source = state.filters.source ? `&source=${state.filters.source}` : "";
|
||||||
const channel = state.filters.channelId ? `&channelId=${state.filters.channelId}` : "";
|
const channel = state.filters.channelId ? `&channelId=${state.filters.channelId}` : "";
|
||||||
|
const author = state.filters.authorId ? `&authorId=${state.filters.authorId}` : "";
|
||||||
|
const aspect = state.filters.aspect ? `&aspect=${state.filters.aspect}` : "";
|
||||||
|
const resolution = state.filters.resolution ? `&resolution=${state.filters.resolution}` : "";
|
||||||
|
const after = state.filters.after ? `&after=${state.filters.after}T00:00:00.000Z` : "";
|
||||||
|
const before = state.filters.before ? `&before=${state.filters.before}T23:59:59.999Z` : "";
|
||||||
const items = await window.VK.api.get(
|
const items = await window.VK.api.get(
|
||||||
`/api/media?limit=60&offset=${state.offset}${kind}${source}${channel}`
|
`/api/media?limit=60&offset=${state.offset}${kind}${source}${channel}${author}${aspect}${resolution}${after}${before}`
|
||||||
);
|
);
|
||||||
state.items = state.items.concat(items);
|
state.items = state.items.concat(items);
|
||||||
state.offset += items.length;
|
state.offset += items.length;
|
||||||
@@ -163,23 +191,28 @@
|
|||||||
renderGrid();
|
renderGrid();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function startScrape(mode) {
|
async function startScrape(channelId, guildId, mode) {
|
||||||
const select = overlay.querySelector("#media-channel");
|
if (!channelId) {
|
||||||
if (!select?.value) {
|
setStatus("no channel selected");
|
||||||
setStatus("no text channel available to scrape");
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const channel = state.channels.find(entry => entry.id === select.value);
|
|
||||||
try {
|
try {
|
||||||
const scrape = await window.VK.api.post("/api/media/scrape", {
|
const scrape = await window.VK.api.post("/api/media/scrape", {
|
||||||
channelId: select.value,
|
channelId,
|
||||||
guildId: channel?.guildId ?? null,
|
guildId: guildId ?? null,
|
||||||
mode
|
mode,
|
||||||
|
backend: guildId ? "auto" : "puppet"
|
||||||
});
|
});
|
||||||
setStatus(`started ${scrape.id} (${mode})`);
|
setStatus(`started ${scrape.id} (${mode})`);
|
||||||
|
scrapeModal.hidden = true;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (error.code === "MEDIA_LIMIT" || String(error.message).includes("MEDIA_LIMIT")) {
|
||||||
|
setStatus(`near the storage limit: ${error.message}`);
|
||||||
|
} else {
|
||||||
setStatus(`start failed: ${error.message}`);
|
setStatus(`start failed: ${error.message}`);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
void loadScrapes();
|
void loadScrapes();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -189,7 +222,23 @@
|
|||||||
const rows = active.length ? active : state.scrapes.slice(0, 5);
|
const rows = active.length ? active : state.scrapes.slice(0, 5);
|
||||||
for (const scrape of rows) {
|
for (const scrape of rows) {
|
||||||
const row = el("div", "vk-media-scrape");
|
const row = el("div", "vk-media-scrape");
|
||||||
row.append(el("span", null, `${scrape.channelId} · ${scrape.state} · ${scrape.pages}p · ${scrape.messages}m`));
|
const media = scrape.media ?? {};
|
||||||
|
|
||||||
|
if (scrape.state === "running" || scrape.state === "queued") {
|
||||||
|
row.append(el("span", "vk-media-pulse"));
|
||||||
|
}
|
||||||
|
|
||||||
|
row.append(
|
||||||
|
el(
|
||||||
|
"span",
|
||||||
|
null,
|
||||||
|
`${channelLabel(scrape.channelId)} · ${scrape.state} · ${scrape.pages}p · ${scrape.messages}m · ` +
|
||||||
|
`img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0}` +
|
||||||
|
(scrape.downloaded ? ` · files ${scrape.downloaded}` : "") +
|
||||||
|
(scrape.previewed ? ` · previews ${scrape.previewed}` : "")
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
if (active.includes(scrape)) {
|
if (active.includes(scrape)) {
|
||||||
row.append(
|
row.append(
|
||||||
button("Stop", async () => {
|
button("Stop", async () => {
|
||||||
@@ -197,7 +246,10 @@
|
|||||||
void loadScrapes();
|
void loadScrapes();
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
} else {
|
||||||
|
row.append(el("span", "vk-media-status", durationLabel(scrape)));
|
||||||
}
|
}
|
||||||
|
|
||||||
scrapeList.append(row);
|
scrapeList.append(row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -211,6 +263,11 @@
|
|||||||
card.target = "_blank";
|
card.target = "_blank";
|
||||||
card.rel = "noreferrer";
|
card.rel = "noreferrer";
|
||||||
|
|
||||||
|
const postedBy = [item.authorName, item.postedAt ? new Date(item.postedAt).toLocaleString() : null]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" · ");
|
||||||
|
if (postedBy) card.title = postedBy;
|
||||||
|
|
||||||
if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) {
|
if ((item.kind === "image" || item.kind === "gif") && /^(https?:|\/)/.test(source)) {
|
||||||
const img = document.createElement("img");
|
const img = document.createElement("img");
|
||||||
img.loading = "lazy";
|
img.loading = "lazy";
|
||||||
@@ -225,6 +282,29 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function filterSelect(options, value, onChange) {
|
||||||
|
const select = document.createElement("select");
|
||||||
|
select.className = "vk-chip vk-chip-off";
|
||||||
|
for (const [optionValue, label] of options) {
|
||||||
|
const option = document.createElement("option");
|
||||||
|
option.value = optionValue;
|
||||||
|
option.textContent = label;
|
||||||
|
select.append(option);
|
||||||
|
}
|
||||||
|
select.value = value;
|
||||||
|
select.addEventListener("change", () => onChange(select.value));
|
||||||
|
return select;
|
||||||
|
}
|
||||||
|
|
||||||
|
function dateInput(value, onChange) {
|
||||||
|
const input = document.createElement("input");
|
||||||
|
input.type = "date";
|
||||||
|
input.className = "vk-media-input";
|
||||||
|
input.value = value ?? "";
|
||||||
|
input.addEventListener("change", () => onChange(input.value));
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
function renderToolbar() {
|
function renderToolbar() {
|
||||||
toolbar.replaceChildren();
|
toolbar.replaceChildren();
|
||||||
if (state.view === "media") {
|
if (state.view === "media") {
|
||||||
@@ -239,19 +319,8 @@
|
|||||||
toolbar.append(chip);
|
toolbar.append(chip);
|
||||||
}
|
}
|
||||||
|
|
||||||
const select = document.createElement("select");
|
toolbar.append(button("New scrape…", openScrapeModal));
|
||||||
select.id = "media-channel";
|
toolbar.append(button("History", openHistoryModal));
|
||||||
select.className = "vk-chip vk-chip-off";
|
|
||||||
for (const channel of state.channels) {
|
|
||||||
const option = document.createElement("option");
|
|
||||||
option.value = channel.id;
|
|
||||||
option.textContent = channel.label;
|
|
||||||
select.append(option);
|
|
||||||
}
|
|
||||||
toolbar.append(select);
|
|
||||||
|
|
||||||
toolbar.append(button("Start links", () => void startScrape("links")));
|
|
||||||
toolbar.append(button("Start download", () => void startScrape("download")));
|
|
||||||
|
|
||||||
const sourceSelect = document.createElement("select");
|
const sourceSelect = document.createElement("select");
|
||||||
sourceSelect.className = "vk-chip vk-chip-off";
|
sourceSelect.className = "vk-chip vk-chip-off";
|
||||||
@@ -296,6 +365,69 @@
|
|||||||
void loadMedia(true);
|
void loadMedia(true);
|
||||||
});
|
});
|
||||||
toolbar.append(channelSelect);
|
toolbar.append(channelSelect);
|
||||||
|
|
||||||
|
const authorOptions = [["", "Any user"]].concat(
|
||||||
|
(state.authors ?? [])
|
||||||
|
.filter(author => author.id)
|
||||||
|
.map(author => [author.id, `${author.name} (${author.count})`])
|
||||||
|
);
|
||||||
|
toolbar.append(
|
||||||
|
filterSelect(authorOptions, state.filters.authorId, value => {
|
||||||
|
state.filters.authorId = value;
|
||||||
|
void loadMedia(true);
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
toolbar.append(
|
||||||
|
filterSelect(
|
||||||
|
[
|
||||||
|
["", "Any ratio"],
|
||||||
|
["vertical", "Vertical"],
|
||||||
|
["square", "1:1"],
|
||||||
|
["wide43", "4:3"],
|
||||||
|
["wide169", "16:9"],
|
||||||
|
["ultrawide", "21:9"],
|
||||||
|
["other", "Other"]
|
||||||
|
],
|
||||||
|
state.filters.aspect,
|
||||||
|
value => {
|
||||||
|
state.filters.aspect = value;
|
||||||
|
void loadMedia(true);
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
toolbar.append(
|
||||||
|
filterSelect(
|
||||||
|
[
|
||||||
|
["", "Any quality"],
|
||||||
|
["sd", "SD (<=480p)"],
|
||||||
|
["hd", "HD (720p)"],
|
||||||
|
["fhd", "FHD (1080p)"],
|
||||||
|
["qhd", "QHD (1440p)"],
|
||||||
|
["uhd", "4K+"],
|
||||||
|
["unknown", "Unknown"]
|
||||||
|
],
|
||||||
|
state.filters.resolution,
|
||||||
|
value => {
|
||||||
|
state.filters.resolution = value;
|
||||||
|
void loadMedia(true);
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
toolbar.append(
|
||||||
|
el("span", "vk-media-status", "From"),
|
||||||
|
dateInput(state.filters.after, value => {
|
||||||
|
state.filters.after = value;
|
||||||
|
void loadMedia(true);
|
||||||
|
}),
|
||||||
|
el("span", "vk-media-status", "To"),
|
||||||
|
dateInput(state.filters.before, value => {
|
||||||
|
state.filters.before = value;
|
||||||
|
void loadMedia(true);
|
||||||
|
})
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
toolbar.append(el("span", null, "Slopbot is the dedicated bot core. Setup: docs/slopbot.md."));
|
toolbar.append(el("span", null, "Slopbot is the dedicated bot core. Setup: docs/slopbot.md."));
|
||||||
}
|
}
|
||||||
@@ -314,6 +446,7 @@
|
|||||||
: "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)";
|
: "slopbot is disabled or not logged in (VESKSLOP_SLOPBOT_ENABLED=1 and a token)";
|
||||||
} else {
|
} else {
|
||||||
void loadChannels().then(renderToolbar);
|
void loadChannels().then(renderToolbar);
|
||||||
|
void loadAuthors().then(renderToolbar);
|
||||||
void loadMedia(true);
|
void loadMedia(true);
|
||||||
}
|
}
|
||||||
void loadScrapes();
|
void loadScrapes();
|
||||||
@@ -332,6 +465,172 @@
|
|||||||
document.querySelector("#scope-guild")?.addEventListener("click", hide);
|
document.querySelector("#scope-guild")?.addEventListener("click", hide);
|
||||||
document.querySelector("#scope-dm")?.addEventListener("click", hide);
|
document.querySelector("#scope-dm")?.addEventListener("click", hide);
|
||||||
|
|
||||||
|
// ---- Scrape popup: server or group-chat trees ----
|
||||||
|
|
||||||
|
const scrapeModal = document.createElement("div");
|
||||||
|
scrapeModal.className = "vk-media-modal";
|
||||||
|
scrapeModal.hidden = true;
|
||||||
|
scrapeModal.innerHTML = `
|
||||||
|
<div class="vk-media-modal-box vk-media-wide">
|
||||||
|
<strong>New scrape</strong>
|
||||||
|
<div class="vk-media-toolbar">
|
||||||
|
<button id="scrape-tab-server" class="vk-chip vk-chip-on" type="button">Server</button>
|
||||||
|
<button id="scrape-tab-group" class="vk-chip vk-chip-off" type="button">Group chat</button>
|
||||||
|
<span id="scrape-hint" class="vk-media-status"></span>
|
||||||
|
</div>
|
||||||
|
<div id="scrape-tree" class="vk-media-tree"></div>
|
||||||
|
<div class="vk-media-toolbar">
|
||||||
|
<button id="scrape-cancel" class="btn btn-secondary vk-small-btn" type="button">Close</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
document.body.append(scrapeModal);
|
||||||
|
|
||||||
|
let scrapeTab = "server";
|
||||||
|
|
||||||
|
function channelRow(channelId, guildId, label) {
|
||||||
|
const row = el("div", "vk-media-channel");
|
||||||
|
row.append(el("span", "vk-media-channel-name", label));
|
||||||
|
row.append(button("Links", () => void startScrape(channelId, guildId, "links")));
|
||||||
|
row.append(button("Download", () => void startScrape(channelId, guildId, "download")));
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderScrapeTree() {
|
||||||
|
const tree = scrapeModal.querySelector("#scrape-tree");
|
||||||
|
const hint = scrapeModal.querySelector("#scrape-hint");
|
||||||
|
tree.replaceChildren();
|
||||||
|
|
||||||
|
if (scrapeTab === "group") {
|
||||||
|
const groups = state.dms.filter(dm => dm.type === 3);
|
||||||
|
hint.textContent = groups.length ? "Group chats use the puppet account." : "No cached group chats yet.";
|
||||||
|
for (const dm of groups) {
|
||||||
|
const names = (dm.recipients ?? [])
|
||||||
|
.map(recipient => recipient.username)
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(", ");
|
||||||
|
tree.append(channelRow(dm.id, null, dm.name || names || dm.id));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
hint.textContent = state.guilds.length ? "" : "No guilds cached yet.";
|
||||||
|
for (const guild of state.guilds) {
|
||||||
|
const group = el("div", "vk-media-tree-guild");
|
||||||
|
const head = el("button", "vk-media-tree-head", `> ${guild.name}`);
|
||||||
|
head.type = "button";
|
||||||
|
const children = el("div", "vk-media-tree-children");
|
||||||
|
children.hidden = true;
|
||||||
|
|
||||||
|
head.addEventListener("click", () => {
|
||||||
|
children.hidden = !children.hidden;
|
||||||
|
head.textContent = `${children.hidden ? ">" : "v"} ${guild.name}`;
|
||||||
|
});
|
||||||
|
|
||||||
|
group.append(head, children);
|
||||||
|
for (const channel of guild.channels ?? []) {
|
||||||
|
if (channel.type !== 0 && channel.type !== 5) continue;
|
||||||
|
children.append(channelRow(channel.id, guild.id, `#${channel.name}`));
|
||||||
|
}
|
||||||
|
tree.append(group);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function openScrapeModal() {
|
||||||
|
renderScrapeTree();
|
||||||
|
scrapeModal.hidden = false;
|
||||||
|
void loadChannels().then(renderScrapeTree);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [tab, id] of [
|
||||||
|
["server", "#scrape-tab-server"],
|
||||||
|
["group", "#scrape-tab-group"]
|
||||||
|
]) {
|
||||||
|
scrapeModal.querySelector(id).addEventListener("click", () => {
|
||||||
|
scrapeTab = tab;
|
||||||
|
scrapeModal.querySelector("#scrape-tab-server").className =
|
||||||
|
tab === "server" ? "vk-chip vk-chip-on" : "vk-chip vk-chip-off";
|
||||||
|
scrapeModal.querySelector("#scrape-tab-group").className =
|
||||||
|
tab === "group" ? "vk-chip vk-chip-on" : "vk-chip vk-chip-off";
|
||||||
|
renderScrapeTree();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
scrapeModal.querySelector("#scrape-cancel").addEventListener("click", () => {
|
||||||
|
scrapeModal.hidden = true;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- History popup ----
|
||||||
|
|
||||||
|
const historyModal = document.createElement("div");
|
||||||
|
historyModal.className = "vk-media-modal";
|
||||||
|
historyModal.hidden = true;
|
||||||
|
historyModal.innerHTML = `
|
||||||
|
<div class="vk-media-modal-box vk-media-wide">
|
||||||
|
<strong>Scrape history</strong>
|
||||||
|
<div id="history-list" class="vk-media-modal-list"></div>
|
||||||
|
<div class="vk-media-toolbar">
|
||||||
|
<button id="history-close" class="btn btn-secondary vk-small-btn" type="button">Close</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
document.body.append(historyModal);
|
||||||
|
|
||||||
|
function durationLabel(scrape) {
|
||||||
|
const start = new Date(scrape.createdAt).getTime();
|
||||||
|
const end = scrape.finishedAt ? new Date(scrape.finishedAt).getTime() : Date.now();
|
||||||
|
const seconds = Math.max(0, Math.round((end - start) / 1000));
|
||||||
|
if (seconds < 60) return `${seconds}s`;
|
||||||
|
const minutes = Math.floor(seconds / 60);
|
||||||
|
if (minutes < 60) return `${minutes}m ${seconds % 60}s`;
|
||||||
|
return `${Math.floor(minutes / 60)}h ${minutes % 60}m`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function channelLabel(channelId) {
|
||||||
|
return state.channels.find(channel => channel.id === channelId)?.label ?? channelId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openHistoryModal() {
|
||||||
|
const list = historyModal.querySelector("#history-list");
|
||||||
|
list.replaceChildren(el("div", null, "loading..."));
|
||||||
|
historyModal.hidden = false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const [scrapes, storage] = await Promise.all([
|
||||||
|
window.VK.api.get("/api/media/scrapes"),
|
||||||
|
window.VK.api.get("/api/media/storage")
|
||||||
|
]);
|
||||||
|
await loadChannels();
|
||||||
|
|
||||||
|
list.replaceChildren();
|
||||||
|
const bytesById = new Map((storage?.scrapes ?? []).map(entry => [entry.id, entry.bytes]));
|
||||||
|
|
||||||
|
for (const scrape of scrapes) {
|
||||||
|
const media = scrape.media ?? {};
|
||||||
|
const row = el("div", "vk-media-history-row");
|
||||||
|
row.append(el("strong", null, channelLabel(scrape.channelId)));
|
||||||
|
row.append(
|
||||||
|
el(
|
||||||
|
"span",
|
||||||
|
null,
|
||||||
|
`${scrape.state} · started ${new Date(scrape.createdAt).toLocaleString()} · ` +
|
||||||
|
`took ${durationLabel(scrape)} · ${scrape.pages}p/${scrape.messages}m · ` +
|
||||||
|
`img ${media.image ?? 0} gif ${media.gif ?? 0} vid ${media.video ?? 0} aud ${media.audio ?? 0} · ` +
|
||||||
|
`${formatBytes(bytesById.get(scrape.id) ?? 0)} · files ${scrape.downloaded ?? 0} · previews ${scrape.previewed ?? 0}`
|
||||||
|
)
|
||||||
|
);
|
||||||
|
list.append(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!scrapes.length) list.append(el("div", null, "No scrapes yet."));
|
||||||
|
} catch (error) {
|
||||||
|
list.replaceChildren(el("div", null, `history failed: ${error.message}`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
historyModal.querySelector("#history-close").addEventListener("click", () => {
|
||||||
|
historyModal.hidden = true;
|
||||||
|
});
|
||||||
|
|
||||||
// ---- Settings: media storage ----
|
// ---- Settings: media storage ----
|
||||||
|
|
||||||
function formatBytes(bytes) {
|
function formatBytes(bytes) {
|
||||||
|
|||||||
@@ -829,12 +829,37 @@ function buildServer(): McpServer {
|
|||||||
.describe("Where the item came from"),
|
.describe("Where the item came from"),
|
||||||
scrapeId: z.string().optional().describe("Restrict to one scrape"),
|
scrapeId: z.string().optional().describe("Restrict to one scrape"),
|
||||||
channelId: z.string().optional().describe("Restrict to one source channel"),
|
channelId: z.string().optional().describe("Restrict to one source channel"),
|
||||||
|
authorId: z.string().optional().describe("Restrict to one poster"),
|
||||||
|
aspect: z
|
||||||
|
.enum(["vertical", "square", "wide43", "wide169", "ultrawide", "other"])
|
||||||
|
.optional()
|
||||||
|
.describe("Aspect-ratio bucket"),
|
||||||
|
resolution: z
|
||||||
|
.enum(["sd", "hd", "fhd", "qhd", "uhd", "unknown"])
|
||||||
|
.optional()
|
||||||
|
.describe("Resolution bucket by shortest side"),
|
||||||
|
after: z.string().optional().describe("Only items posted at or after this ISO time"),
|
||||||
|
before: z.string().optional().describe("Only items posted at or before this ISO time"),
|
||||||
limit: z.number().int().min(1).max(500).optional(),
|
limit: z.number().int().min(1).max(500).optional(),
|
||||||
offset: z.number().int().min(0).optional()
|
offset: z.number().int().min(0).optional()
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
async ({ kind, source, scrapeId, channelId, limit, offset }) =>
|
async ({ kind, source, scrapeId, channelId, authorId, aspect, resolution, after, before, limit, offset }) =>
|
||||||
jsonText(media.queryMedia({ kind, source, scrapeId, channelId, limit, offset }))
|
jsonText(
|
||||||
|
media.queryMedia({
|
||||||
|
kind,
|
||||||
|
source,
|
||||||
|
scrapeId,
|
||||||
|
channelId,
|
||||||
|
authorId,
|
||||||
|
aspect,
|
||||||
|
resolution,
|
||||||
|
after,
|
||||||
|
before,
|
||||||
|
limit,
|
||||||
|
offset
|
||||||
|
})
|
||||||
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
server.registerTool(
|
server.registerTool(
|
||||||
|
|||||||
@@ -93,6 +93,12 @@ export interface MediaItemResult {
|
|||||||
channelId: string;
|
channelId: string;
|
||||||
kind: MediaKind;
|
kind: MediaKind;
|
||||||
source: string | null;
|
source: string | null;
|
||||||
|
authorId: string | null;
|
||||||
|
authorName: string | null;
|
||||||
|
postedAt: string | null;
|
||||||
|
width: number | null;
|
||||||
|
height: number | null;
|
||||||
|
aspectRatio: number | null;
|
||||||
url: string;
|
url: string;
|
||||||
localUrl: string | null;
|
localUrl: string | null;
|
||||||
thumbUrl: string | null;
|
thumbUrl: string | null;
|
||||||
@@ -337,6 +343,29 @@ export class MediaService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Distinct posters across every scrape, biggest first. */
|
||||||
|
listAuthors(): Array<{ id: string | null; name: string | null; count: number }> {
|
||||||
|
const totals = new Map<string, { id: string | null; name: string | null; count: number }>();
|
||||||
|
|
||||||
|
for (const meta of this.engine.list()) {
|
||||||
|
if (meta.state === "deleted") continue;
|
||||||
|
|
||||||
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
||||||
|
try {
|
||||||
|
for (const author of store.authors()) {
|
||||||
|
const key = author.id ?? author.name ?? "unknown";
|
||||||
|
const current = totals.get(key);
|
||||||
|
if (current) current.count += author.count;
|
||||||
|
else totals.set(key, { ...author });
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
store.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...totals.values()].sort((a, b) => b.count - a.count);
|
||||||
|
}
|
||||||
|
|
||||||
/** One media row inside a scrape, with its scrape and channel for context. */
|
/** One media row inside a scrape, with its scrape and channel for context. */
|
||||||
getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null {
|
getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null {
|
||||||
const meta = this.engine.get(scrapeId);
|
const meta = this.engine.get(scrapeId);
|
||||||
@@ -358,6 +387,11 @@ export class MediaService {
|
|||||||
source?: string;
|
source?: string;
|
||||||
scrapeId?: string;
|
scrapeId?: string;
|
||||||
channelId?: string;
|
channelId?: string;
|
||||||
|
authorId?: string;
|
||||||
|
aspect?: string;
|
||||||
|
resolution?: string;
|
||||||
|
after?: string;
|
||||||
|
before?: string;
|
||||||
limit?: number;
|
limit?: number;
|
||||||
offset?: number;
|
offset?: number;
|
||||||
} = {}
|
} = {}
|
||||||
@@ -377,6 +411,11 @@ export class MediaService {
|
|||||||
const rows = store.queryMedia({
|
const rows = store.queryMedia({
|
||||||
kind: options.kind,
|
kind: options.kind,
|
||||||
source: options.source,
|
source: options.source,
|
||||||
|
authorId: options.authorId,
|
||||||
|
aspect: options.aspect,
|
||||||
|
resolution: options.resolution,
|
||||||
|
after: options.after,
|
||||||
|
before: options.before,
|
||||||
limit: limit + offset
|
limit: limit + offset
|
||||||
});
|
});
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
@@ -388,6 +427,12 @@ export class MediaService {
|
|||||||
channelId: meta.channelId,
|
channelId: meta.channelId,
|
||||||
kind: row.kind,
|
kind: row.kind,
|
||||||
source: row.source,
|
source: row.source,
|
||||||
|
authorId: row.author_id,
|
||||||
|
authorName: row.author_name,
|
||||||
|
postedAt: row.posted_at,
|
||||||
|
width: row.width,
|
||||||
|
height: row.height,
|
||||||
|
aspectRatio: row.aspect_ratio,
|
||||||
url: row.url,
|
url: row.url,
|
||||||
localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null,
|
localUrl: row.file_path ? `/api/media/${meta.id}/${row.id}/file` : null,
|
||||||
thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
|
thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
|
||||||
|
|||||||
+27
-1
@@ -799,10 +799,36 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
|||||||
? String(req.query.source)
|
? String(req.query.source)
|
||||||
: undefined;
|
: undefined;
|
||||||
const channelId = req.query.channelId ? String(req.query.channelId) : undefined;
|
const channelId = req.query.channelId ? String(req.query.channelId) : undefined;
|
||||||
|
const authorId = req.query.authorId ? String(req.query.authorId) : undefined;
|
||||||
|
const aspect = req.query.aspect ? String(req.query.aspect) : undefined;
|
||||||
|
const resolution = req.query.resolution ? String(req.query.resolution) : undefined;
|
||||||
|
const after = req.query.after ? String(req.query.after) : undefined;
|
||||||
|
const before = req.query.before ? String(req.query.before) : undefined;
|
||||||
const limit = clampLimit(req.query.limit, 60, 500);
|
const limit = clampLimit(req.query.limit, 60, 500);
|
||||||
const offset = Math.max(0, Number(req.query.offset ?? 0) || 0);
|
const offset = Math.max(0, Number(req.query.offset ?? 0) || 0);
|
||||||
|
|
||||||
res.json({ ok: true, result: deps.media.queryMedia({ kind, source, channelId, limit, offset }) });
|
res.json({
|
||||||
|
ok: true,
|
||||||
|
result: deps.media.queryMedia({
|
||||||
|
kind,
|
||||||
|
source,
|
||||||
|
channelId,
|
||||||
|
authorId,
|
||||||
|
aspect,
|
||||||
|
resolution,
|
||||||
|
after,
|
||||||
|
before,
|
||||||
|
limit,
|
||||||
|
offset
|
||||||
|
})
|
||||||
|
});
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
app.get(
|
||||||
|
"/api/media/authors",
|
||||||
|
wrap(async (_req, res) => {
|
||||||
|
res.json({ ok: true, result: deps.media.listAuthors() });
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -210,16 +210,32 @@ function mapAttachments(message: any) {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Dimensions of the first embed part that has them: video, image, then thumbnail. */
|
||||||
|
function embedSize(embed: any) {
|
||||||
|
for (const part of [embed?.video, embed?.image, embed?.thumbnail]) {
|
||||||
|
if (part && Number.isFinite(part.width) && Number.isFinite(part.height)) {
|
||||||
|
return { width: part.width, height: part.height };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { width: null, height: null };
|
||||||
|
}
|
||||||
|
|
||||||
function mapEmbeds(message: any) {
|
function mapEmbeds(message: any) {
|
||||||
return (message?.embeds ?? []).map((embed: any) => ({
|
return (message?.embeds ?? []).map((embed: any) => {
|
||||||
|
const size = embedSize(embed);
|
||||||
|
|
||||||
|
return {
|
||||||
type: embed?.type ?? null,
|
type: embed?.type ?? null,
|
||||||
url: embed?.url ?? null,
|
url: embed?.url ?? null,
|
||||||
title: embed?.title ?? null,
|
title: embed?.title ?? null,
|
||||||
provider: embed?.provider?.name ?? null,
|
provider: embed?.provider?.name ?? null,
|
||||||
videoUrl: embedUrl(embed?.video),
|
videoUrl: embedUrl(embed?.video),
|
||||||
imageUrl: embedUrl(embed?.image),
|
imageUrl: embedUrl(embed?.image),
|
||||||
thumbnailUrl: embedUrl(embed?.thumbnail)
|
thumbnailUrl: embedUrl(embed?.thumbnail),
|
||||||
}));
|
width: size.width,
|
||||||
|
height: size.height
|
||||||
|
};
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function mapStickers(message: any) {
|
function mapStickers(message: any) {
|
||||||
|
|||||||
Reference in New Issue
Block a user