Add the media downloader with hash storage and local file serving

media-scrape/src/store.ts

@@ -98,6 +98,7 @@ export function uniqueScrapeId(root: string, channelId: string, now = new Date()
 export interface MediaRow {
     id: number;
     kind: MediaKind;
+    source: string | null;
     url: string;
     provider: string | null;
     content_type: string | null;
@@ -109,6 +110,10 @@ export interface MediaRow {
     preview_url: string | null;
     file_hash: string | null;
     preview_hash: string | null;
+    file_bytes: number | null;
+    preview_bytes: number | null;
+    file_path: string | null;
+    download_error: string | null;
     created_at: string;
 }
 
@@ -124,6 +129,7 @@ CREATE TABLE IF NOT EXISTS messages (
 CREATE TABLE IF NOT EXISTS media (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     kind TEXT NOT NULL,
+    source TEXT,
     url TEXT NOT NULL UNIQUE,
     provider TEXT,
     content_type TEXT,
@@ -137,6 +143,8 @@ CREATE TABLE IF NOT EXISTS media (
     preview_hash TEXT,
     file_bytes INTEGER,
     preview_bytes INTEGER,
+    file_path TEXT,
+    download_error TEXT,
     created_at TEXT NOT NULL
 );
 
@@ -167,6 +175,9 @@ export class ScrapeStore {
 
         if (!names.has("file_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN file_bytes INTEGER");
         if (!names.has("preview_bytes")) this.db.exec("ALTER TABLE media ADD COLUMN preview_bytes INTEGER");
+        if (!names.has("source")) this.db.exec("ALTER TABLE media ADD COLUMN source TEXT");
+        if (!names.has("file_path")) this.db.exec("ALTER TABLE media ADD COLUMN file_path TEXT");
+        if (!names.has("download_error")) this.db.exec("ALTER TABLE media ADD COLUMN download_error TEXT");
     }
 
     close(): void {
@@ -180,7 +191,7 @@ export class ScrapeStore {
             "INSERT OR REPLACE INTO messages (id, author_id, author_name, timestamp, content) VALUES (?, ?, ?, ?, ?)"
         );
         const insertMedia = this.db.prepare(
-            "INSERT OR IGNORE INTO media (kind, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+            "INSERT OR IGNORE INTO media (kind, source, url, provider, content_type, size, width, height, duration_secs, downloaded, preview_url, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
         );
         const findMedia = this.db.prepare("SELECT id FROM media WHERE url = ?");
         const linkSource = this.db.prepare(
@@ -202,6 +213,7 @@ export class ScrapeStore {
                 for (const item of extractMedia(message)) {
                     const result = insertMedia.run(
                         item.kind,
+                        item.source,
                         item.url,
                         item.provider,
                         item.contentType,
@@ -255,6 +267,30 @@ export class ScrapeStore {
         return result;
     }
 
+    /** Media rows that still need bytes: never downloaded and not skipped. */
+    pendingDownloads(limit = 5000): MediaRow[] {
+        return this.db
+            .prepare(
+                "SELECT * FROM media WHERE downloaded = 0 AND file_path IS NULL AND download_error IS NULL AND (source IS NULL OR source != 'embed') ORDER BY id LIMIT ?"
+            )
+            .all(limit) as unknown as MediaRow[];
+    }
+
+    markDownloaded(
+        id: number,
+        file: { path: string; hash: string; bytes: number }
+    ): void {
+        this.db
+            .prepare(
+                "UPDATE media SET downloaded = 1, file_path = ?, file_hash = ?, file_bytes = ?, download_error = NULL WHERE id = ?"
+            )
+            .run(file.path, file.hash, file.bytes, id);
+    }
+
+    markDownloadError(id: number, error: string): void {
+        this.db.prepare("UPDATE media SET download_error = ? WHERE id = ?").run(error.slice(0, 200), id);
+    }
+
     getMedia(id: number): MediaRow | null {
         const row = this.db.prepare("SELECT * FROM media WHERE id = ?").get(id) as unknown as
             | MediaRow