Backfill previews for paused scrapes and prefer richer dedupe rows

2026-09-16 14:02:11 +02:00
parent 2309393e3c
commit 96fe17b4c9
8 changed files with 259 additions and 99 deletions
@@ -14,6 +14,7 @@ import { ogImageFromHtml, resolveEmbedPreview } from "../src/embed-preview.js";
 import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
+import { betterCandidate } from "../src/dedupe.js";
 import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
 import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
@@ -861,6 +862,90 @@ void (async () => {
        assert.equal(existsSync(join(dir, `${poster?.hash}.webp`)), true);
    });

+    await test("backfillPreviews fills rows left by a preview-less walk", async () => {
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("800", {
+                        attachments: [attachment({ filename: "later.png", contentType: "image/png" })],
+                        embeds: [
+                            {
+                                type: "video",
+                                url: "https://www.youtube.com/watch?v=x",
+                                title: null,
+                                provider: "YouTube",
+                                videoUrl: "https://www.youtube.com/embed/x",
+                                imageUrl: null,
+                                thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
+                            }
+                        ]
+                    })
+                ]
+            }
+        ]);
+
+        const engine = new ScrapeEngine({
+            rootDir: root,
+            pageSize: 2,
+            pageMinMs: 0,
+            pageMaxMs: 0,
+            sleep: async () => {},
+            random: () => 0,
+            log: () => {},
+            previews: false,
+            download: {
+                fetchImpl: async () =>
+                    new Response("webp", { status: 200, headers: { "content-type": "image/webp" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "132", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+        assert.equal(engine.get(started.id)?.previewed, undefined);
+
+        const added = await engine.backfillPreviews(started.id);
+        assert.equal(added, 2);
+
+        const store = new ScrapeStore(join(root, started.id));
+        assert.ok(store.queryMedia({ kind: "image" })[0].preview_hash);
+        assert.ok(store.queryMedia({ kind: "video" })[0].preview_hash);
+        store.close();
+    });
+
+    await test("dedupe prefers the richer row for a URL", () => {
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: false },
+                { downloaded: false, hasPreview: true }
+            ),
+            true
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: true },
+                { downloaded: false, hasPreview: false }
+            ),
+            false
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: true, hasPreview: false },
+                { downloaded: false, hasPreview: true }
+            ),
+            false
+        );
+        assert.equal(
+            betterCandidate(
+                { downloaded: false, hasPreview: true },
+                { downloaded: true, hasPreview: false }
+            ),
+            true
+        );
+    });
+
    await test("snowflake helpers round-trip dates", () => {
        const id = snowflakeFromIso("2026-09-16T00:00:00.000Z");
        assert.ok(id);