diff --git a/CURRENT.md b/CURRENT.md index 545f0ed..e24777a 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,13 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C33 Puppet history pagination. The core's `message.fetch` read the client store, which + keeps only the newest messages plus the channel's first — every page was 10 messages and the + walk stopped after one page (a 10-message, 4-media scrape of an ancient group with thousands + of messages). It now fetches through the client's REST layer first (contiguous 100-message + pages, `transport: "rest"`), store as fallback. Rebuilt and restarted the managed client; + live re-run of channel `591000745164406784`: 21+ pages / 2,100+ messages and climbing, with + hundreds of images/videos being indexed. - [x] C32 Group-chat clarity and puppet refresh. Group DM scrapes already worked (live-verified against the running daemon: `zıbb & fred` and a 1:1 DM, backend `puppet`, done, previews cached); the Group chat tab had shown an empty list while the core was offline. It now says so diff --git a/docs/core-bridge.md b/docs/core-bridge.md index d0e3a47..dda6793 100644 --- a/docs/core-bridge.md +++ b/docs/core-bridge.md @@ -68,7 +68,7 @@ Event line: | `dm.list` | optional `limit` | Cached DM and group DM channels with recipients, icons, and avatars. Cache-only. | | `message.send` | `channelId`, `content` | `{ channelId, messageId, sent: true }` | | `message.read` | `channelId`, `limit` | `{ cached, messages }`. Cache-only. Messages carry the author avatar, attachment metadata (id, filename, URL, content type, size, dimensions, duration), embeds (video, image, and thumbnail URLs plus their dimensions), sticker items, forwarded snapshots, a voice-message flag, `editedAt`, and `replyTo`. | -| `message.fetch` | `channelId`, `limit`, optional `before` or `after` | `{ fetched, messages }`. Fetches from Discord. Same message shape. | +| `message.fetch` | `channelId`, `limit`, optional `before` or `after` | `{ fetched, transport, messages }`. Fetches from Discord through the client's REST layer, so pages are full and contiguous; the client store (which keeps only the newest messages plus the channel's first) is the fallback. `transport` is `rest` or `store`. Same message shape. | | `reaction.add` | `channelId`, `messageId`, `emoji` | `{ added: true }` | | `dm.send` | `userId`, `content` | `{ userId, channelId, sent: true }` | diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 8bf0bc6..172c5e2 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -18,7 +18,7 @@ interface ScrapeBackend { | Backend | Implementation | | --- | --- | | `slopbot` | discord.js REST (`channels.messages.fetch`), bucket-aware client | -| `puppet` | The core action `message.fetch` over the bridge. Expired URLs refresh with `before: id+1`, which returns exactly the target message | +| `puppet` | The core action `message.fetch` over the bridge: the client's REST layer paginates (100/page, contiguous), with the client store as fallback. Expired URLs refresh with `before: id+1`, which returns exactly the target message | The daemon picks `slopbot` when the channel is a guild channel and slopbot is configured and in the guild; otherwise `puppet`. A scrape records the backend it used. The walk can resume on the same diff --git a/src/renderer/puppet/actions.ts b/src/renderer/puppet/actions.ts index dafad0f..7929a4d 100644 --- a/src/renderer/puppet/actions.ts +++ b/src/renderer/puppet/actions.ts @@ -23,6 +23,7 @@ import { MessageStore, PresenceStore, ReadStateStore, + RestAPI, UserStore, VoiceStateStore } from "@vencord/types/webpack/common"; @@ -961,24 +962,63 @@ export async function handlePuppetAction(action: string, params: Record { + const inRange = (message: any) => { if (after && compareIds(message.id, after) <= 0) return false; if (before && compareIds(message.id, before) >= 0) return false; return true; - }); + }; + + const normalize = (list: any[]) => { + const seen = new Set(); + return list + .filter(message => inRange(message) && !seen.has(message.id) && seen.add(message.id)) + .sort((a, b) => compareIds(a.id, b.id)) + .slice(-Number(limit)); + }; + + // Preferred: the client's REST layer paginates properly. The store keeps only + // the newest messages plus the channel's first, so reading it back yields tiny, + // non-contiguous pages. + let messages: any[] | null = null; + let transport = "rest"; + + try { + const query: Record = { limit: Number(limit) }; + if (before) query.before = before; + if (after) query.after = after; + + const response = await RestAPI.get({ + url: `/channels/${channelId}/messages`, + query: { ...query } + }); + const list = Array.isArray(response) + ? response + : Array.isArray((response as any)?.body) + ? (response as any).body + : null; + if (list) messages = normalize(list); + } catch { + messages = null; + } + + if (messages === null) { + transport = "store"; + await fetchMessages({ + channelId, + limit: Number(limit), + ...(before ? { before } : {}), + ...(after ? { after } : {}) + }); + + const all = MessageStore.getMessages(channelId)?._array ?? []; + messages = normalize(all); + } return ok({ channelId, - fetched: filtered.length, - messages: filtered.slice(-Number(limit)).map(mapMessage) + fetched: messages.length, + transport, + messages: messages.map(mapMessage) }); }