Add message links, suspect retries, and Discord preview recovery

media-scrape/test/run.ts

@@ -24,6 +24,7 @@ import { BackfillQueue } from "../src/backfill.js";
 import { TitleLane } from "../src/titles.js";
 import { extractMedia, normalizeYouTubeUrl } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
+import { messageUrlFor } from "../src/discord-links.js";
 import { betterCandidate, sortCandidates } from "../src/dedupe.js";
 import { aspectCondition, PROVIDER_FILTERS, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
@@ -1436,6 +1437,116 @@ void (async () => {
         reopened.close();
     });
 
+    await test("message links build for guilds and DMs", () => {
+        assert.equal(messageUrlFor("123", "456", "789"), "https://discord.com/channels/123/456/789");
+        assert.equal(messageUrlFor(null, "456", "789"), "https://discord.com/channels/@me/456/789");
+        assert.equal(messageUrlFor("123", "456", null), null);
+    });
+
+    await test("suspect rows park, due, and confirm", () => {
+        const dir = join(root, "suspects");
+        mkdirSync(dir, { recursive: true });
+        const store = new ScrapeStore(dir);
+        store.insertPage([
+            message("1700", {
+                attachments: [attachment({ filename: "gone.mp4", contentType: "video/mp4" })]
+            })
+        ]);
+
+        const row = store.queryMedia({ kind: "video" })[0];
+        const now = Date.now();
+        store.markSuspect(row.id, "http 404", now + 10_000);
+
+        assert.equal(store.previewCounts().suspect, 1);
+        assert.equal(store.previewCounts().unverified, 0);
+        assert.equal(store.pendingPreviews(10, true).length, 0);
+        assert.equal(store.dueSuspects(10, now).length, 0);
+        assert.equal(store.dueSuspects(10, now + 20_000).length, 1);
+
+        store.confirmSuspect(row.id, "still http 404 after retry");
+        assert.equal(store.previewCounts().suspect, 0);
+        assert.equal(store.previewCounts().unverified, 1);
+        assert.equal(store.dueSuspects(10, now + 20_000).length, 0);
+
+        store.markSuspect(row.id, "http 503", now);
+        assert.equal(store.previewCounts().suspect, 1);
+        assert.equal(store.previewCounts().unverified, 0);
+        assert.equal(store.dueSuspects(10, now).length, 1);
+        store.close();
+    });
+
+    await test("discord 404s become suspects and the delayed retry resolves them", async () => {
+        let failing = true;
+        const backend = new FakeBackend([
+            {
+                messages: [
+                    message("1701", {
+                        attachments: [attachment({ filename: "clip.mp4", contentType: "video/mp4" })]
+                    })
+                ]
+            }
+        ]);
+        const engine = engineFor(root, backend, {
+            suspectRetryMs: 0,
+            download: {
+                fetchImpl: async () =>
+                    failing
+                        ? new Response("gone", { status: 404 })
+                        : new Response("png", { status: 200, headers: { "content-type": "image/png" } }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+
+        const started = await engine.start({ channelId: "138", guildId: null, mode: "links", backend });
+        await engine.processQueue();
+
+        const first = await engine.previewBatch(started.id, 5);
+        assert.equal(first.suspect, 1);
+        assert.equal(first.skipped, 0);
+
+        // Normal passes leave suspects alone.
+        assert.equal((await engine.previewBatch(started.id, 5)).suspect, 0);
+
+        failing = false;
+        const retry = await engine.previewSuspects(started.id, 5);
+        assert.equal(retry.cached, 1);
+
+        const store = new ScrapeStore(join(root, started.id));
+        assert.equal(store.previewCounts().cached, 1);
+        assert.equal(store.previewCounts().suspect, 0);
+        store.close();
+
+        // A retry that fails again is confirmed unverified.
+        const dead = new FakeBackend([
+            {
+                messages: [
+                    message("1702", {
+                        attachments: [attachment({ filename: "dead.mp4", contentType: "video/mp4" })]
+                    })
+                ]
+            }
+        ]);
+        const deadEngine = engineFor(root, dead, {
+            suspectRetryMs: 0,
+            download: {
+                fetchImpl: async () => new Response("gone", { status: 404 }),
+                minGapMs: 0,
+                maxGapMs: 0,
+                sleep: async () => {}
+            }
+        });
+        const deadScrape = await deadEngine.start({ channelId: "139", guildId: null, mode: "links", backend: dead });
+        await deadEngine.processQueue();
+        assert.equal((await deadEngine.previewBatch(deadScrape.id, 5)).suspect, 1);
+        assert.equal((await deadEngine.previewSuspects(deadScrape.id, 5)).unverified, 1);
+        const deadStore = new ScrapeStore(join(root, deadScrape.id));
+        assert.equal(deadStore.previewCounts().unverified, 1);
+        assert.equal(deadStore.pendingPreviews(10, true).length, 0);
+        deadStore.close();
+    });
+
     await test("pendingTitles selects YouTube rows without a title", () => {
         const dir = join(root, "pending-titles");
         mkdirSync(dir, { recursive: true });