Deduplicate media across scrapes and polish the viewer controls

2026-09-16 13:05:31 +02:00
parent 1b5fcfed2f
commit d3f9d20190
12 changed files with 198 additions and 85 deletions
@@ -4,6 +4,7 @@
        <meta charset="utf-8" />
        <meta name="viewport" content="width=device-width, initial-scale=1" />
        <title>veskslop debug</title>
+        <link rel="icon" href="data:," />
        <link rel="stylesheet" href="nocturne.css" />
        <link rel="stylesheet" href="style.css" />
        <link rel="stylesheet" href="media.css" />
@@ -179,6 +179,7 @@
 }

 .vk-media-viewer-media {
+    position: relative;
    display: flex;
    flex: 1;
    flex-direction: column;
@@ -188,6 +189,12 @@
    min-height: 0;
 }

+.vk-media-play-big {
+    top: 12px;
+    right: 14px;
+    font-size: 44px;
+}
+
 .vk-media-viewer-media img,
 .vk-media-viewer-media video {
    max-width: 100%;
@@ -417,6 +417,13 @@
        return play;
    }

+    function viewerPlayIcon(item) {
+        const play = el("span", "vk-media-play vk-media-play-big", "▶");
+        play.title = "Play";
+        play.addEventListener("click", () => openViewer(item, true));
+        return play;
+    }
+
    function renderCard(item) {
        const figure = el("figure", "vk-media-figure");
        const media = el("div", "vk-media-media");
@@ -1074,33 +1081,33 @@
        media.replaceChildren();

        const source = item.localUrl || item.thumbUrl || item.url;
-        const embed = play ? embedUrlFor(item) : null;
+        const direct = item.localUrl || (item.source !== "embed" ? item.url : null);
+        const playableFile = !!direct && /\.(mp4|webm|m4v)(\?|$)/i.test(direct);
+        const embed = embedUrlFor(item);
+        const canPlay = !!embed || playableFile;

-        if (embed) {
+        // A play request on something we cannot embed falls back to the external page.
+        if (play && !canPlay) openLinkWarning(item.url);
+
+        if (play && embed) {
            const frame = document.createElement("iframe");
            frame.src = embed;
            frame.allow = "autoplay; fullscreen; picture-in-picture";
            frame.allowFullscreen = true;
            media.append(frame);
-        } else if (item.kind === "video" && /^(https?:|\/)/.test(source)) {
-            if (play) {
-                const video = document.createElement("video");
-                video.controls = true;
-                video.autoplay = true;
-                video.src = source;
-                media.append(video);
-            } else {
-                const img = document.createElement("img");
-                img.src = source;
-                img.alt = item.kind;
-                media.append(img);
-                media.append(button("Play", () => openViewer(item, true)));
-            }
+        } else if (play && playableFile && direct) {
+            const video = document.createElement("video");
+            video.controls = true;
+            video.autoplay = true;
+            video.src = direct;
+            media.append(video);
        } else if (/^(https?:|\/)/.test(source)) {
            const img = document.createElement("img");
            img.src = source;
            img.alt = item.kind;
            media.append(img);
+
+            if (item.kind === "video") media.append(viewerPlayIcon(item));
        } else {
            media.append(el("span", "vk-media-card-kind", item.kind.toUpperCase()));
        }
@@ -1120,9 +1127,9 @@
        link.addEventListener("click", () => openLinkWarning(item.url));
        meta.append(link);

-        meta.append(
-            button("View full", () => window.open(item.localUrl || item.url, "_blank", "noreferrer"))
-        );
+        // "View full" opens the embedded media (player, local file, or preview) in a new tab.
+        const fullUrl = embedUrlFor(item) ?? item.localUrl ?? item.thumbUrl ?? item.url;
+        meta.append(button("View full", () => window.open(fullUrl, "_blank", "noreferrer")));

        viewerModal.hidden = false;
    }
@@ -456,7 +456,8 @@ export class MediaService {
        after?: string;
        before?: string;
    } = {}): { total: number; downloaded: number; cached: number; notDownloaded: number } {
-        const totals = { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 };
+        // Deduplicate by URL across scrapes, newest scrape wins, like queryMedia.
+        const byUrl = new Map<string, { downloaded: boolean; cached: boolean }>();

        for (const meta of this.engine.list()) {
            if (meta.state === "deleted") continue;
@@ -464,16 +465,38 @@ export class MediaService {

            const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
            try {
-                const stats = store.stats(options);
-                totals.total += stats.total;
-                totals.downloaded += stats.downloaded;
-                totals.cached += stats.cached;
-                totals.notDownloaded += stats.notDownloaded;
+                let storeOffset = 0;
+
+                for (;;) {
+                    const rows = store.queryMedia({ ...options, limit: 500, offset: storeOffset });
+                    if (!rows.length) break;
+                    storeOffset += rows.length;
+
+                    for (const row of rows) {
+                        if (byUrl.has(row.url)) continue;
+                        byUrl.set(row.url, {
+                            downloaded: row.downloaded === 1,
+                            cached: row.preview_hash !== null
+                        });
+                    }
+
+                    if (rows.length < 500) break;
+                }
            } finally {
                store.close();
            }
        }

+        const totals = { total: byUrl.size, downloaded: 0, cached: 0, notDownloaded: 0 };
+        for (const entry of byUrl.values()) {
+            if (entry.downloaded) {
+                totals.downloaded++;
+            } else {
+                totals.notDownloaded++;
+                if (entry.cached) totals.cached++;
+            }
+        }
+
        return totals;
    }

@@ -533,6 +556,7 @@ export class MediaService {
        const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
        const offset = Math.max(0, options.offset ?? 0);
        const results: MediaItemResult[] = [];
+        const seen = new Set<string>();
        let skipped = 0;

        for (const meta of this.engine.list()) {
@@ -542,43 +566,61 @@ export class MediaService {

            const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
            try {
-                const rows = store.queryMedia({
-                    kind: options.kind,
-                    source: options.source,
-                    authorId: options.authorId,
-                    aspect: options.aspect,
-                    resolution: options.resolution,
-                    after: options.after,
-                    before: options.before,
-                    limit: limit + offset
-                });
-                for (const row of rows) {
-                    if (skipped++ < offset) continue;
-                    if (results.length >= limit) break;
-                    results.push({
-                        scrapeId: meta.id,
-                        itemId: row.id,
-                        channelId: meta.channelId,
-                        kind: row.kind,
-                        source: row.source,
-                        authorId: row.author_id,
-                        authorName: row.author_name,
-                        postedAt: row.posted_at,
-                        width: row.width,
-                        height: row.height,
-                        aspectRatio: row.aspect_ratio,
-                        contentType: row.content_type,
-                        durationSecs: row.duration_secs,
-                        url: row.url,
-                        localUrl:
-                            row.file_path && !meta.downloadDir
-                                ? `/api/media/${meta.id}/${row.id}/file`
-                                : null,
-                        thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
-                        provider: row.provider,
-                        downloaded: row.downloaded === 1,
-                        createdAt: row.created_at
+                let storeOffset = 0;
+
+                for (;;) {
+                    const rows = store.queryMedia({
+                        kind: options.kind,
+                        source: options.source,
+                        authorId: options.authorId,
+                        aspect: options.aspect,
+                        resolution: options.resolution,
+                        after: options.after,
+                        before: options.before,
+                        limit: 500,
+                        offset: storeOffset
                    });
+                    if (!rows.length) break;
+                    storeOffset += rows.length;
+
+                    for (const row of rows) {
+                        // One card per URL, newest scrape wins.
+                        if (seen.has(row.url)) continue;
+                        seen.add(row.url);
+
+                        if (skipped < offset) {
+                            skipped++;
+                            continue;
+                        }
+                        if (results.length >= limit) break;
+
+                        results.push({
+                            scrapeId: meta.id,
+                            itemId: row.id,
+                            channelId: meta.channelId,
+                            kind: row.kind,
+                            source: row.source,
+                            authorId: row.author_id,
+                            authorName: row.author_name,
+                            postedAt: row.posted_at,
+                            width: row.width,
+                            height: row.height,
+                            aspectRatio: row.aspect_ratio,
+                            contentType: row.content_type,
+                            durationSecs: row.duration_secs,
+                            url: row.url,
+                            localUrl:
+                                row.file_path && !meta.downloadDir
+                                    ? `/api/media/${meta.id}/${row.id}/file`
+                                    : null,
+                            thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
+                            provider: row.provider,
+                            downloaded: row.downloaded === 1,
+                            createdAt: row.created_at
+                        });
+                    }
+
+                    if (results.length >= limit || rows.length < 500) break;
                }
            } finally {
                store.close();