Deduplicate media across scrapes and polish the viewer controls

CURRENT.md

@@ -40,6 +40,12 @@ mobile draft at 390px.
   - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
     cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
     `previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
+  - [x] C20 Cross-scrape dedupe and viewer polish. `queryMedia` and `stats` deduplicate by URL
+    (newest scrape wins), so overlapping scrapes no longer duplicate cards or counters (live: 17
+    items / 17 unique). Viewer: big play icon top-right; play embeds a provider player or a direct
+    mp4/webm/m4v, otherwise opens the external warning; **View full** opens the embedded
+    media/player in a new tab while the link button opens the original page; duration is stored
+    (Discord attachment duration, ffprobe for downloads, 0 normalized to unknown); favicon added.
   - [x] C19 External embed previews and MIME fixes. Embeds with no thumbnail in Discord's data are
     resolved via provider oEmbed/fxtwitter and an `og:image` fallback; preview bytes keep their
     real extension (`preview_ext`), so JPEG thumbnails serve as `image/jpeg` instead of mislabeled

docs/media-scraper.md

@@ -111,6 +111,8 @@ Downloads run after a finished walk when the scrape used `mode: "download"`:
   `<destination>/<scrape-id>/thumbnails/` while the local copy stays for the Media tab.
 - Downloaded videos get a poster frame (`ffmpeg -ss 0.5`, 480px WebP) cached in `thumbs/`, because
   Discord serves no thumbnails for uploaded video files.
+- The ffprobe pass fills missing dimensions and `duration_secs` after a download. Attachment
+  durations come from Discord; the 0 that Discord sends for non-audio files is treated as unknown.
 - Progress counters: `downloaded`, `downloadErrors`.
 
 Previews run after every walk:

docs/media.md

@@ -67,6 +67,10 @@ media-scrape/db/
 - The database is an index. It holds the scrape state, the messages seen, the media items, and the
   link between a media item and its source messages. Source channel, author, and timestamp are
   indexed so filters can be added later without a re-scrape.
+- Scrapes stay self-contained, but the API and UI **deduplicate by URL across scrapes**: one card
+  per URL, the newest scrape wins; stats count unique URLs too. Re-scraping a channel does not
+  duplicate anything in the grid or the counters. (Files are still stored per scrape, so the
+  on-disk size doubles with overlapping scrapes; a shared content store is a possible later step.)
 - Storage uses `node:sqlite` (experimental in Node 22, prints a warning). If its API changes,
   `better-sqlite3` is the fallback; it needs a native build, so that switch is an owner install.
 - The file name in `media/` is the SHA-256 of the bytes, never the original name. The original
@@ -147,9 +151,13 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun
   tiles carry a translucent play icon top-right. Discord serves no thumbnails for uploaded videos,
   so a video without a cached preview renders a first-frame `<video>` in the grid, and downloaded
   videos get an ffmpeg poster cached in `thumbs/`. Clicking a tile opens a viewer: wide media
-  shows the metadata banner below, vertical media shows it to the right; the banner has the full
+  shows the metadata banner below, vertical media shows it to the right, and videos carry a large
-  link (with an external-site warning) and **View full**, and the play action embeds a YouTube,
+  play icon top-right. The play action embeds a YouTube, Vimeo, Twitch, or Streamable player, or
-  Vimeo, Twitch, or Streamable player when possible. The status row reads
+  plays a direct mp4/webm/m4v; when neither is possible it opens the external-site warning, since
+  the page is the only way to watch. **View full** opens the embedded media (player, local file,
+  or cached preview) in a new tab; the full link is a separate button that opens the original
+  page with the warning. Length comes from Discord's attachment duration and, for downloads, an
+  ffprobe probe. The status row reads
   `(Images/Videos) | Downloaded A/B | Cached C/B (excl Downl) | Loaded L/B`, where B is the total
   for the active filters. A scrape strip shows live progress with Stop.
   **New scrape…** (rightmost) opens a popup with Server and Group chat trees (servers expand to

media-scrape/src/download.ts

@@ -12,23 +12,41 @@ import type { MediaRow } from "./store.js";
 
 const execFileAsync = promisify(execFile);
 
-/** Read pixel dimensions with ffprobe. Null when the tool is missing or the file is not media. */
+/** Read dimensions and duration with ffprobe. Null when nothing usable is found. */
 export async function probeDimensions(
     path: string,
     ffprobe = process.env.VESKSLOP_FFPROBE ?? "ffprobe"
-): Promise<{ width: number; height: number } | null> {
+): Promise<{ width: number | null; height: number | null; durationSecs: number | null } | null> {
     try {
         const { stdout } = await execFileAsync(
             ffprobe,
-            ["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "json", path],
+            [
+                "-v",
+                "error",
+                "-select_streams",
+                "v:0",
+                "-show_entries",
+                "stream=width,height:format=duration",
+                "-of",
+                "json",
+                path
+            ],
             { timeout: 15_000 }
         );
-        const stream = JSON.parse(stdout)?.streams?.[0];
+
-        if (Number.isFinite(stream?.width) && Number.isFinite(stream?.height)) {
+        const parsed = JSON.parse(stdout);
-            return { width: stream.width, height: stream.height };
+        const stream = parsed?.streams?.[0];
-        }
+        const duration = Number(parsed?.format?.duration);
+
+        const width = Number.isFinite(stream?.width) ? stream.width : null;
+        const height = Number.isFinite(stream?.height) ? stream.height : null;
+        const durationSecs =
+            Number.isFinite(duration) && duration > 0 ? Math.round(duration * 10) / 10 : null;
+
+        if (width === null && height === null && durationSecs === null) return null;
+        return { width, height, durationSecs };
     } catch {
-        // missing ffprobe, a timeout, or a non-visual file
+        // missing ffprobe, a timeout, or a non-media file
     }
     return null;
 }

media-scrape/src/engine.ts

@@ -59,8 +59,10 @@ export interface ScrapeEngineOptions {
     download?: DownloadOptions;
     /** Cache Discord-hosted image previews after every walk. Default true. */
     previews?: boolean;
-    /** Read dimensions from a downloaded file. Defaults to ffprobe. */
+    /** Read dimensions and duration from a downloaded file. Defaults to ffprobe. */
-    probe?: (path: string) => Promise<{ width: number; height: number } | null>;
+    probe?: (
+        path: string
+    ) => Promise<{ width: number | null; height: number | null; durationSecs: number | null } | null>;
     /** Called when a scrape fails. */
     onError?: (scrapeId: string, message: string) => void;
 }
@@ -320,11 +322,11 @@ export class ScrapeEngine {
                 });
                 meta.downloaded = (meta.downloaded ?? 0) + 1;
 
-                if (row.width === null || row.height === null) {
+                if (row.width === null || row.height === null || row.duration_secs === null) {
-                    const dimensions = await (this.options.probe ?? probeDimensions)(
+                    const info = await (this.options.probe ?? probeDimensions)(
                         join(targetDir, outcome.path)
                     );
-                    if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height);
+                    if (info) store.updateMediaInfo(row.id, info);
                 }
 
                 if (row.kind === "video" && !row.preview_hash) {

media-scrape/src/extract.ts

@@ -151,7 +151,11 @@ function extractFrom(message: MediaMessage): ExtractedMedia[] {
             size: attachment.size,
             width: attachment.width,
             height: attachment.height,
-            durationSecs: attachment.durationSecs,
+            // Discord sends 0 for non-audio attachments; treat it as unknown.
+            durationSecs:
+                Number.isFinite(attachment.durationSecs) && (attachment.durationSecs as number) > 0
+                    ? attachment.durationSecs
+                    : null,
             downloaded: false,
             previewUrl: null
         });

media-scrape/src/store.ts

@@ -362,11 +362,25 @@ export class ScrapeStore {
         this.db.prepare("UPDATE media SET preview_error = ? WHERE id = ?").run(error.slice(0, 200), id);
     }
 
-    /** Fill in dimensions after a download probe. */
+    /** Fill in dimensions and duration after a download probe. */
-    updateDimensions(id: number, width: number, height: number): void {
+    updateMediaInfo(
-        this.db
+        id: number,
-            .prepare("UPDATE media SET width = ?, height = ?, aspect_ratio = ? WHERE id = ?")
+        info: { width?: number | null; height?: number | null; durationSecs?: number | null }
-            .run(width, height, aspectRatioOf(width, height), id);
+    ): void {
+        const sets: string[] = [];
+        const values: Array<number | null> = [];
+
+        if (info.width != null && info.height != null) {
+            sets.push("width = ?", "height = ?", "aspect_ratio = ?");
+            values.push(info.width, info.height, aspectRatioOf(info.width, info.height));
+        }
+        if (info.durationSecs != null) {
+            sets.push("duration_secs = ?");
+            values.push(info.durationSecs);
+        }
+
+        if (!sets.length) return;
+        this.db.prepare(`UPDATE media SET ${sets.join(", ")} WHERE id = ?`).run(...values, id);
     }
 
     /** Distinct posters with their item counts, biggest first. */

media-scrape/test/run.ts

@@ -103,7 +103,7 @@ void (async () => {
                 attachments: [
                     attachment({ filename: "cat.png", contentType: "image/png" }),
                     attachment({ filename: "loop.gif" }),
-                    attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 1234 }),
+                    attachment({ filename: "clip.mp4", contentType: "video/mp4", size: 1234, durationSecs: 0 }),
                     attachment({ filename: "note.ogg", durationSecs: 3.2 })
                 ],
                 embeds: [
@@ -143,6 +143,7 @@ void (async () => {
         assert.equal(items.find(item => item.url.endsWith("/stickers/55.gif"))?.kind, "gif");
         assert.equal(items.some(item => item.url.includes("lottie")), false);
         assert.equal(items.find(item => item.kind === "audio")?.durationSecs, 3.2);
+        assert.equal(items.find(item => item.url.endsWith("clip.mp4"))?.durationSecs, null);
     });
 
     await test("extract finds media in forwarded snapshots", () => {
@@ -491,7 +492,7 @@ void (async () => {
             random: () => 0,
             log: () => {},
             previews: false,
-            probe: async () => ({ width: 1080, height: 1920 }),
+            probe: async () => ({ width: 1080, height: 1920, durationSecs: 12.3 }),
             download: {
                 fetchImpl: async () =>
                     new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
@@ -509,6 +510,7 @@ void (async () => {
         assert.equal(row.width, 1080);
         assert.equal(row.height, 1920);
         assert.equal(row.aspect_ratio, 0.563);
+        assert.equal(row.duration_secs, 12.3);
         store.close();
     });
 

poc/public/index.html

@@ -4,6 +4,7 @@
         <meta charset="utf-8" />
         <meta name="viewport" content="width=device-width, initial-scale=1" />
         <title>veskslop debug</title>
+        <link rel="icon" href="data:," />
         <link rel="stylesheet" href="nocturne.css" />
         <link rel="stylesheet" href="style.css" />
         <link rel="stylesheet" href="media.css" />

poc/public/media.css

@@ -179,6 +179,7 @@
 }
 
 .vk-media-viewer-media {
+    position: relative;
     display: flex;
     flex: 1;
     flex-direction: column;
@@ -188,6 +189,12 @@
     min-height: 0;
 }
 
+.vk-media-play-big {
+    top: 12px;
+    right: 14px;
+    font-size: 44px;
+}
+
 .vk-media-viewer-media img,
 .vk-media-viewer-media video {
     max-width: 100%;

poc/public/media.js

@@ -417,6 +417,13 @@
         return play;
     }
 
+    function viewerPlayIcon(item) {
+        const play = el("span", "vk-media-play vk-media-play-big", "▶");
+        play.title = "Play";
+        play.addEventListener("click", () => openViewer(item, true));
+        return play;
+    }
+
     function renderCard(item) {
         const figure = el("figure", "vk-media-figure");
         const media = el("div", "vk-media-media");
@@ -1074,33 +1081,33 @@
         media.replaceChildren();
 
         const source = item.localUrl || item.thumbUrl || item.url;
-        const embed = play ? embedUrlFor(item) : null;
+        const direct = item.localUrl || (item.source !== "embed" ? item.url : null);
+        const playableFile = !!direct && /\.(mp4|webm|m4v)(\?|$)/i.test(direct);
+        const embed = embedUrlFor(item);
+        const canPlay = !!embed || playableFile;
 
-        if (embed) {
+        // A play request on something we cannot embed falls back to the external page.
+        if (play && !canPlay) openLinkWarning(item.url);
+
+        if (play && embed) {
             const frame = document.createElement("iframe");
             frame.src = embed;
             frame.allow = "autoplay; fullscreen; picture-in-picture";
             frame.allowFullscreen = true;
             media.append(frame);
-        } else if (item.kind === "video" && /^(https?:|\/)/.test(source)) {
+        } else if (play && playableFile && direct) {
-            if (play) {
+            const video = document.createElement("video");
-                const video = document.createElement("video");
+            video.controls = true;
-                video.controls = true;
+            video.autoplay = true;
-                video.autoplay = true;
+            video.src = direct;
-                video.src = source;
+            media.append(video);
-                media.append(video);
-            } else {
-                const img = document.createElement("img");
-                img.src = source;
-                img.alt = item.kind;
-                media.append(img);
-                media.append(button("Play", () => openViewer(item, true)));
-            }
         } else if (/^(https?:|\/)/.test(source)) {
             const img = document.createElement("img");
             img.src = source;
             img.alt = item.kind;
             media.append(img);
+
+            if (item.kind === "video") media.append(viewerPlayIcon(item));
         } else {
             media.append(el("span", "vk-media-card-kind", item.kind.toUpperCase()));
         }
@@ -1120,9 +1127,9 @@
         link.addEventListener("click", () => openLinkWarning(item.url));
         meta.append(link);
 
-        meta.append(
+        // "View full" opens the embedded media (player, local file, or preview) in a new tab.
-            button("View full", () => window.open(item.localUrl || item.url, "_blank", "noreferrer"))
+        const fullUrl = embedUrlFor(item) ?? item.localUrl ?? item.thumbUrl ?? item.url;
-        );
+        meta.append(button("View full", () => window.open(fullUrl, "_blank", "noreferrer")));
 
         viewerModal.hidden = false;
     }

poc/src/media.ts

@@ -456,7 +456,8 @@ export class MediaService {
         after?: string;
         before?: string;
     } = {}): { total: number; downloaded: number; cached: number; notDownloaded: number } {
-        const totals = { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 };
+        // Deduplicate by URL across scrapes, newest scrape wins, like queryMedia.
+        const byUrl = new Map<string, { downloaded: boolean; cached: boolean }>();
 
         for (const meta of this.engine.list()) {
             if (meta.state === "deleted") continue;
@@ -464,16 +465,38 @@ export class MediaService {
 
             const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
             try {
-                const stats = store.stats(options);
+                let storeOffset = 0;
-                totals.total += stats.total;
+
-                totals.downloaded += stats.downloaded;
+                for (;;) {
-                totals.cached += stats.cached;
+                    const rows = store.queryMedia({ ...options, limit: 500, offset: storeOffset });
-                totals.notDownloaded += stats.notDownloaded;
+                    if (!rows.length) break;
+                    storeOffset += rows.length;
+
+                    for (const row of rows) {
+                        if (byUrl.has(row.url)) continue;
+                        byUrl.set(row.url, {
+                            downloaded: row.downloaded === 1,
+                            cached: row.preview_hash !== null
+                        });
+                    }
+
+                    if (rows.length < 500) break;
+                }
             } finally {
                 store.close();
             }
         }
 
+        const totals = { total: byUrl.size, downloaded: 0, cached: 0, notDownloaded: 0 };
+        for (const entry of byUrl.values()) {
+            if (entry.downloaded) {
+                totals.downloaded++;
+            } else {
+                totals.notDownloaded++;
+                if (entry.cached) totals.cached++;
+            }
+        }
+
         return totals;
     }
 
@@ -533,6 +556,7 @@ export class MediaService {
         const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
         const offset = Math.max(0, options.offset ?? 0);
         const results: MediaItemResult[] = [];
+        const seen = new Set<string>();
         let skipped = 0;
 
         for (const meta of this.engine.list()) {
@@ -542,43 +566,61 @@ export class MediaService {
 
             const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
             try {
-                const rows = store.queryMedia({
+                let storeOffset = 0;
-                    kind: options.kind,
+
-                    source: options.source,
+                for (;;) {
-                    authorId: options.authorId,
+                    const rows = store.queryMedia({
-                    aspect: options.aspect,
+                        kind: options.kind,
-                    resolution: options.resolution,
+                        source: options.source,
-                    after: options.after,
+                        authorId: options.authorId,
-                    before: options.before,
+                        aspect: options.aspect,
-                    limit: limit + offset
+                        resolution: options.resolution,
-                });
+                        after: options.after,
-                for (const row of rows) {
+                        before: options.before,
-                    if (skipped++ < offset) continue;
+                        limit: 500,
-                    if (results.length >= limit) break;
+                        offset: storeOffset
-                    results.push({
-                        scrapeId: meta.id,
-                        itemId: row.id,
-                        channelId: meta.channelId,
-                        kind: row.kind,
-                        source: row.source,
-                        authorId: row.author_id,
-                        authorName: row.author_name,
-                        postedAt: row.posted_at,
-                        width: row.width,
-                        height: row.height,
-                        aspectRatio: row.aspect_ratio,
-                        contentType: row.content_type,
-                        durationSecs: row.duration_secs,
-                        url: row.url,
-                        localUrl:
-                            row.file_path && !meta.downloadDir
-                                ? `/api/media/${meta.id}/${row.id}/file`
-                                : null,
-                        thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
-                        provider: row.provider,
-                        downloaded: row.downloaded === 1,
-                        createdAt: row.created_at
                     });
+                    if (!rows.length) break;
+                    storeOffset += rows.length;
+
+                    for (const row of rows) {
+                        // One card per URL, newest scrape wins.
+                        if (seen.has(row.url)) continue;
+                        seen.add(row.url);
+
+                        if (skipped < offset) {
+                            skipped++;
+                            continue;
+                        }
+                        if (results.length >= limit) break;
+
+                        results.push({
+                            scrapeId: meta.id,
+                            itemId: row.id,
+                            channelId: meta.channelId,
+                            kind: row.kind,
+                            source: row.source,
+                            authorId: row.author_id,
+                            authorName: row.author_name,
+                            postedAt: row.posted_at,
+                            width: row.width,
+                            height: row.height,
+                            aspectRatio: row.aspect_ratio,
+                            contentType: row.content_type,
+                            durationSecs: row.duration_secs,
+                            url: row.url,
+                            localUrl:
+                                row.file_path && !meta.downloadDir
+                                    ? `/api/media/${meta.id}/${row.id}/file`
+                                    : null,
+                            thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
+                            provider: row.provider,
+                            downloaded: row.downloaded === 1,
+                            createdAt: row.created_at
+                        });
+                    }
+
+                    if (results.length >= limit || rows.length < 500) break;
                 }
             } finally {
                 store.close();