Drive provider filters from the archive dataset

media-scrape/src/filter-presets.ts

@@ -48,3 +48,23 @@ export function resolutionCondition(preset: string): string | null {
             return null;
     }
 }
+
+/**
+ * The providers that dominate the archive, for the "Do not include" shortcuts.
+ * Aggregated 2026-09-17 over media-scrape/db (`provider` column, distinct URLs; discord folds in
+ * cdn.discordapp.com and media.discordapp.net). Matching is a case-insensitive substring on
+ * `provider`, so "twitter" also covers fxtwitter.com. The 4chan hosts stay out: the default URL
+ * host exclusion already hides them.
+ */
+export const PROVIDER_FILTERS = [
+    "discord",
+    "youtube",
+    "tenor",
+    "twitch",
+    "twitter",
+    "steam",
+    "medal",
+    "sketchfab",
+    "gfycat",
+    "imgur"
+] as const;

media-scrape/src/index.ts

@@ -10,6 +10,7 @@ export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape }
 export { isoFromSnowflake, snowflakeFromIso } from "./snowflake.js";
 export {
     ASPECT_PRESETS,
+    PROVIDER_FILTERS,
     RESOLUTION_PRESETS,
     aspectCondition,
     resolutionCondition,

media-scrape/test/run.ts

@@ -23,7 +23,7 @@ import { BackfillQueue } from "../src/backfill.js";
 import { extractMedia } from "../src/extract.js";
 import { PuppetBackend } from "../src/puppet-backend.js";
 import { betterCandidate, sortCandidates } from "../src/dedupe.js";
-import { aspectCondition, resolutionCondition } from "../src/filter-presets.js";
+import { aspectCondition, PROVIDER_FILTERS, resolutionCondition } from "../src/filter-presets.js";
 import { isNearLimit, planRetention } from "../src/retention.js";
 import { isoFromSnowflake, snowflakeFromIso } from "../src/snowflake.js";
 import {
@@ -1347,6 +1347,20 @@ void (async () => {
         store.close();
     });
 
+    await test("provider filters are the curated archive list", () => {
+        assert.ok(PROVIDER_FILTERS.length >= 8 && PROVIDER_FILTERS.length <= 12, PROVIDER_FILTERS.length);
+        assert.equal(new Set(PROVIDER_FILTERS).size, PROVIDER_FILTERS.length);
+
+        for (const value of PROVIDER_FILTERS) {
+            assert.equal(value, value.toLowerCase());
+            assert.ok(!value.includes(" ") && !value.includes("."), `host-like entry: ${value}`);
+        }
+
+        for (const required of ["discord", "youtube", "twitch", "tenor"]) {
+            assert.ok((PROVIDER_FILTERS as readonly string[]).includes(required), required);
+        }
+    });
+
     await test("store exclusion, size, and length filters", () => {
         const dir = join(root, "exclude-test");
         mkdirSync(dir, { recursive: true });