Deduplicate media across scrapes and polish the viewer controls

2026-09-16 13:05:31 +02:00
parent 1b5fcfed2f
commit d3f9d20190
12 changed files with 198 additions and 85 deletions
@@ -456,7 +456,8 @@ export class MediaService {
        after?: string;
        before?: string;
    } = {}): { total: number; downloaded: number; cached: number; notDownloaded: number } {
-        const totals = { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 };
+        // Deduplicate by URL across scrapes, newest scrape wins, like queryMedia.
+        const byUrl = new Map<string, { downloaded: boolean; cached: boolean }>();

        for (const meta of this.engine.list()) {
            if (meta.state === "deleted") continue;
@@ -464,16 +465,38 @@ export class MediaService {

            const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
            try {
-                const stats = store.stats(options);
-                totals.total += stats.total;
-                totals.downloaded += stats.downloaded;
-                totals.cached += stats.cached;
-                totals.notDownloaded += stats.notDownloaded;
+                let storeOffset = 0;
+
+                for (;;) {
+                    const rows = store.queryMedia({ ...options, limit: 500, offset: storeOffset });
+                    if (!rows.length) break;
+                    storeOffset += rows.length;
+
+                    for (const row of rows) {
+                        if (byUrl.has(row.url)) continue;
+                        byUrl.set(row.url, {
+                            downloaded: row.downloaded === 1,
+                            cached: row.preview_hash !== null
+                        });
+                    }
+
+                    if (rows.length < 500) break;
+                }
            } finally {
                store.close();
            }
        }

+        const totals = { total: byUrl.size, downloaded: 0, cached: 0, notDownloaded: 0 };
+        for (const entry of byUrl.values()) {
+            if (entry.downloaded) {
+                totals.downloaded++;
+            } else {
+                totals.notDownloaded++;
+                if (entry.cached) totals.cached++;
+            }
+        }
+
        return totals;
    }

@@ -533,6 +556,7 @@ export class MediaService {
        const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
        const offset = Math.max(0, options.offset ?? 0);
        const results: MediaItemResult[] = [];
+        const seen = new Set<string>();
        let skipped = 0;

        for (const meta of this.engine.list()) {
@@ -542,43 +566,61 @@ export class MediaService {

            const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
            try {
-                const rows = store.queryMedia({
-                    kind: options.kind,
-                    source: options.source,
-                    authorId: options.authorId,
-                    aspect: options.aspect,
-                    resolution: options.resolution,
-                    after: options.after,
-                    before: options.before,
-                    limit: limit + offset
-                });
-                for (const row of rows) {
-                    if (skipped++ < offset) continue;
-                    if (results.length >= limit) break;
-                    results.push({
-                        scrapeId: meta.id,
-                        itemId: row.id,
-                        channelId: meta.channelId,
-                        kind: row.kind,
-                        source: row.source,
-                        authorId: row.author_id,
-                        authorName: row.author_name,
-                        postedAt: row.posted_at,
-                        width: row.width,
-                        height: row.height,
-                        aspectRatio: row.aspect_ratio,
-                        contentType: row.content_type,
-                        durationSecs: row.duration_secs,
-                        url: row.url,
-                        localUrl:
-                            row.file_path && !meta.downloadDir
-                                ? `/api/media/${meta.id}/${row.id}/file`
-                                : null,
-                        thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
-                        provider: row.provider,
-                        downloaded: row.downloaded === 1,
-                        createdAt: row.created_at
+                let storeOffset = 0;
+
+                for (;;) {
+                    const rows = store.queryMedia({
+                        kind: options.kind,
+                        source: options.source,
+                        authorId: options.authorId,
+                        aspect: options.aspect,
+                        resolution: options.resolution,
+                        after: options.after,
+                        before: options.before,
+                        limit: 500,
+                        offset: storeOffset
                    });
+                    if (!rows.length) break;
+                    storeOffset += rows.length;
+
+                    for (const row of rows) {
+                        // One card per URL, newest scrape wins.
+                        if (seen.has(row.url)) continue;
+                        seen.add(row.url);
+
+                        if (skipped < offset) {
+                            skipped++;
+                            continue;
+                        }
+                        if (results.length >= limit) break;
+
+                        results.push({
+                            scrapeId: meta.id,
+                            itemId: row.id,
+                            channelId: meta.channelId,
+                            kind: row.kind,
+                            source: row.source,
+                            authorId: row.author_id,
+                            authorName: row.author_name,
+                            postedAt: row.posted_at,
+                            width: row.width,
+                            height: row.height,
+                            aspectRatio: row.aspect_ratio,
+                            contentType: row.content_type,
+                            durationSecs: row.duration_secs,
+                            url: row.url,
+                            localUrl:
+                                row.file_path && !meta.downloadDir
+                                    ? `/api/media/${meta.id}/${row.id}/file`
+                                    : null,
+                            thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
+                            provider: row.provider,
+                            downloaded: row.downloaded === 1,
+                            createdAt: row.created_at
+                        });
+                    }
+
+                    if (results.length >= limit || rows.length < 500) break;
                }
            } finally {
                store.close();