Add the media scrape engine, store, and puppet walker

2026-09-16 08:03:21 +02:00
parent 92429f1e83
commit 4f953cefdc
12 changed files with 1086 additions and 4 deletions
@@ -0,0 +1,90 @@
+/*
+ * Shared types for the media archive. See docs/media.md and docs/media-scraper.md.
+ */
+
+export type MediaKind = "image" | "gif" | "video" | "audio";
+
+export const MEDIA_KINDS: MediaKind[] = ["image", "gif", "video", "audio"];
+
+export type ScrapeState =
+    | "queued"
+    | "running"
+    | "paused"
+    | "done"
+    | "failed"
+    | "pending-delete"
+    | "deleting"
+    | "deleted";
+
+export type ScrapeMode = "links" | "download";
+
+export type BackendName = "puppet" | "slopbot";
+
+export interface CoreAttachment {
+    id: string | null;
+    filename: string;
+    url: string | null;
+    contentType: string | null;
+    size: number | null;
+    width: number | null;
+    height: number | null;
+    durationSecs: number | null;
+}
+
+export interface CoreEmbed {
+    type: string | null;
+    url: string | null;
+    title: string | null;
+    provider: string | null;
+    videoUrl: string | null;
+    imageUrl: string | null;
+    thumbnailUrl: string | null;
+}
+
+export interface CoreSticker {
+    id: string | null;
+    name: string | null;
+    formatType: number | null;
+}
+
+/** The part of a core message the extractor reads. Forwarded snapshots use the same shape. */
+export interface MediaMessage {
+    attachments: CoreAttachment[];
+    embeds: CoreEmbed[];
+    stickerItems: CoreSticker[];
+    voiceMessage: boolean;
+}
+
+/** A message as shaped by the core bridge (`message.fetch`, `message.read`). */
+export interface CoreMessage extends MediaMessage {
+    id: string;
+    author: { id: string; username: string; avatar: string | null };
+    content: string;
+    timestamp: string;
+    editedAt: string | null;
+    replyTo: { id: string; authorId: string | null; authorName: string | null } | null;
+    forwarded: Array<MediaMessage & { content: string; timestamp: string | null }>;
+}
+
+/** `scrape.json`: the state of one scrape. The directory is the scrape's identity. */
+export interface ScrapeMeta {
+    id: string;
+    channelId: string;
+    guildId: string | null;
+    backend: BackendName;
+    mode: ScrapeMode;
+    state: ScrapeState;
+    /** The oldest message ID seen. The next page is fetched before it. */
+    cursor: string | null;
+    createdAt: string;
+    updatedAt: string;
+    pages: number;
+    messages: number;
+    media: Record<MediaKind, number>;
+    errors: number;
+    lastError: string | null;
+}
+
+export function emptyMediaCounts(): Record<MediaKind, number> {
+    return { image: 0, gif: 0, video: 0, audio: 0 };
+}