diff --git a/CURRENT.md b/CURRENT.md index d523d01..545f0ed 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,11 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C32 Group-chat clarity and puppet refresh. Group DM scrapes already worked (live-verified + against the running daemon: `zıbb & fred` and a 1:1 DM, backend `puppet`, done, previews + cached); the Group chat tab had shown an empty list while the core was offline. It now says so + explicitly and offers Retry. Expired attachment URLs in DM/group-DM scrapes refresh through + the core bridge (`message.fetch` with `before: id+1`, which returns exactly the target). - [x] C31 Batched previews. The preview pass now runs in batches (default 25) with a cooldown (default 60 s), configurable via `VESKSLOP_MEDIA_PREVIEW_BATCH` and `VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`; batching counts rows, not successes. Live log: diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 70d1e5b..8bf0bc6 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -18,7 +18,7 @@ interface ScrapeBackend { | Backend | Implementation | | --- | --- | | `slopbot` | discord.js REST (`channels.messages.fetch`), bucket-aware client | -| `puppet` | The core action `message.fetch` over the bridge | +| `puppet` | The core action `message.fetch` over the bridge. Expired URLs refresh with `before: id+1`, which returns exactly the target message | The daemon picks `slopbot` when the channel is a guild channel and slopbot is configured and in the guild; otherwise `puppet`. A scrape records the backend it used. The walk can resume on the same diff --git a/poc/public/media.js b/poc/public/media.js index 7c9b035..a01118a 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -118,8 +118,10 @@ document.querySelector("#scope-bot")?.classList.remove("active"); } - async function loadChannels() { - if (state.loadedChannels) return; + async function loadChannels(force = false) { + if (state.loadedChannels && !force) return; + state.dmsError = null; + try { const data = await window.VK.api.get("/api/state"); state.guilds = data?.guilds ?? []; @@ -150,14 +152,16 @@ label: `${dm.name || names || "DM"} (${dm.type === 3 ? "group DM" : "DM"})` }); } - } catch { + } catch (error) { state.dms = []; + state.dmsError = error.message; } state.loadedChannels = true; - } catch { + } catch (error) { state.guilds = []; state.channels = []; + state.dmsError = error.message; } } @@ -844,6 +848,16 @@ tree.replaceChildren(); if (scrapeTab === "group") { + if (state.dmsError) { + hint.textContent = `Group chats need the core client: ${state.dmsError}`; + tree.append( + button("Retry", () => { + void loadChannels(true).then(renderScrapeTree); + }) + ); + return; + } + const groups = state.dms.filter(dm => dm.type === 3); hint.textContent = groups.length ? "Group chats use the puppet account." : "No cached group chats yet."; for (const dm of groups) { @@ -881,7 +895,7 @@ function openScrapeModal() { renderScrapeTree(); scrapeModal.hidden = false; - void loadChannels().then(renderScrapeTree); + void loadChannels(true).then(renderScrapeTree); } for (const [tab, id] of [ diff --git a/poc/src/media.ts b/poc/src/media.ts index 2f426e4..4c6480b 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -425,63 +425,97 @@ export class MediaService { const meta = this.engine.get(scrapeId); const item = this.getMediaItem(scrapeId, itemId); if (!meta || !item) return null; - if (meta.backend !== "slopbot" || !meta.guildId || !this.bot) return null; - return this.refreshUrls(meta.id, meta.channelId, item); + return this.refreshUrls(meta, item); } - /** Re-fetch the source message and update expired URLs. Returns the fresh playable URL. */ - private async refreshUrls( - scrapeId: string, - channelId: string, - item: MediaRow - ): Promise { + /** + * Re-fetch the source message and update expired URLs. Returns the fresh playable URL. + * Slopbot scrapes use the bot REST; DM and group-DM scrapes go through the core bridge. + */ + private async refreshUrls(meta: ScrapeMeta, item: MediaRow): Promise { try { - const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); + const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); let messageId: string | null = null; try { messageId = store.sourceMessageId(item.id); } finally { store.close(); } - if (!messageId || !this.bot) return null; - - const channel = await this.bot.channels.fetch(channelId); - if (!channel?.isTextBased()) return null; - const message = await channel.messages.fetch(messageId); + if (!messageId) return null; const path = (url: string) => url.split("?")[0]; - const attachment = [...message.attachments.values()].find( - candidate => path(candidate.url) === path(item.url) - ); - if (attachment) { - const update = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); + const saveAndReturn = (update: { url?: string; playUrl?: string | null }, fresh: string) => { + const writer = new ScrapeStore(scrapeDir(this.rootDir, meta.id)); try { - update.updateUrls(item.id, { url: attachment.url }); + writer.updateUrls(item.id, update); } finally { - update.close(); + writer.close(); + } + return fresh; + }; + + if (this.bot && meta.guildId && meta.backend === "slopbot") { + const channel = await this.bot.channels.fetch(meta.channelId); + if (channel?.isTextBased()) { + const message = await channel.messages.fetch(messageId); + + const attachment = [...message.attachments.values()].find( + candidate => path(candidate.url) === path(item.url) + ); + if (attachment) return saveAndReturn({ url: attachment.url }, attachment.url); + + const embed = message.embeds.find( + candidate => + (item.play_url && + candidate.video?.url && + path(candidate.video.url) === path(item.play_url)) || + (candidate.video?.url && path(candidate.url ?? "") === path(item.url)) + ); + if (embed?.video?.url) { + return saveAndReturn( + { playUrl: embed.video.url, url: embed.url ?? item.url }, + embed.video.url + ); + } } - return attachment.url; } - const embed = message.embeds.find( - candidate => - (item.play_url && candidate.video?.url && path(candidate.video.url) === path(item.play_url)) || - (candidate.video?.url && path(candidate.url ?? "") === path(item.url)) - ); - if (embed?.video?.url) { - const update = new ScrapeStore(scrapeDir(this.rootDir, scrapeId)); - try { - update.updateUrls(item.id, { playUrl: embed.video.url, url: embed.url ?? item.url }); - } finally { - update.close(); + // The puppet path: the newest message below id+1 is exactly the target message. + const core = this.deps.getCore(); + if (core && this.deps.isCoreConnected()) { + const result = (await core.fetchMessages(meta.channelId, 1, { + before: (BigInt(messageId) + 1n).toString() + })) as { messages?: Array> }; + const message = + result?.messages?.find(candidate => candidate?.id === messageId) ?? + result?.messages?.[0]; + + if (message) { + const attachment = (message.attachments ?? []).find( + (candidate: any) => candidate?.url && path(candidate.url) === path(item.url) + ); + if (attachment?.url) return saveAndReturn({ url: attachment.url }, attachment.url); + + const embed = (message.embeds ?? []).find( + (candidate: any) => + (item.play_url && + candidate?.videoUrl && + path(candidate.videoUrl) === path(item.play_url)) || + (candidate?.videoUrl && path(candidate.url ?? "") === path(item.url)) + ); + if (embed?.videoUrl) { + return saveAndReturn( + { playUrl: embed.videoUrl, url: embed.url ?? item.url }, + embed.videoUrl + ); + } } - return embed.video.url; } return null; } catch (error) { - this.deps.log?.(`refresh failed for ${scrapeId}:${item.id}: ${String(error)}`); + this.deps.log?.(`refresh failed for ${meta.id}:${item.id}: ${String(error)}`); return null; } }