zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clarify group-chat scraping and refresh puppet URLs via the core

Browse Source
This commit is contained in:
Fisto
2026-09-16 23:57:47 +02:00
parent 6c500f0cc5
commit 031da7d8b4
4 changed files with 94 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CURRENT.md
+5
View File
@@ -40,6 +40,11 @@ mobile draft at 390px.
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
- [x] C32 Group-chat clarity and puppet refresh. Group DM scrapes already worked (live-verified
against the running daemon: `zıbb & fred` and a 1:1 DM, backend `puppet`, done, previews
cached); the Group chat tab had shown an empty list while the core was offline. It now says so
explicitly and offers Retry. Expired attachment URLs in DM/group-DM scrapes refresh through
the core bridge (`message.fetch` with `before: id+1`, which returns exactly the target).
- [x] C31 Batched previews. The preview pass now runs in batches (default 25) with a cooldown
(default 60 s), configurable via `VESKSLOP_MEDIA_PREVIEW_BATCH` and
`VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS`; batching counts rows, not successes. Live log:
docs/media-scraper.md
+1 -1
View File
@@ -18,7 +18,7 @@ interface ScrapeBackend {
| Backend | Implementation |
| --- | --- |
| `slopbot` | discord.js REST (`channels.messages.fetch`), bucket-aware client |
| `puppet` | The core action `message.fetch` over the bridge |
| `puppet` | The core action `message.fetch` over the bridge. Expired URLs refresh with `before: id+1`, which returns exactly the target message |
The daemon picks `slopbot` when the channel is a guild channel and slopbot is configured and in the
guild; otherwise `puppet`. A scrape records the backend it used. The walk can resume on the same
poc/public/media.js
+19 -5
View File
@@ -118,8 +118,10 @@
document.querySelector("#scope-bot")?.classList.remove("active");
}
async function loadChannels() {
if (state.loadedChannels) return;
async function loadChannels(force = false) {
if (state.loadedChannels && !force) return;
state.dmsError = null;
try {
const data = await window.VK.api.get("/api/state");
state.guilds = data?.guilds ?? [];
@@ -150,14 +152,16 @@
label: `${dm.name || names || "DM"} (${dm.type === 3 ? "group DM" : "DM"})`
});
}
} catch {
} catch (error) {
state.dms = [];
state.dmsError = error.message;
}
state.loadedChannels = true;
} catch {
} catch (error) {
state.guilds = [];
state.channels = [];
state.dmsError = error.message;
}
}
@@ -844,6 +848,16 @@
tree.replaceChildren();
if (scrapeTab === "group") {
if (state.dmsError) {
hint.textContent = `Group chats need the core client: ${state.dmsError}`;
tree.append(
button("Retry", () => {
void loadChannels(true).then(renderScrapeTree);
})
);
return;
}
const groups = state.dms.filter(dm => dm.type === 3);
hint.textContent = groups.length ? "Group chats use the puppet account." : "No cached group chats yet.";
for (const dm of groups) {
@@ -881,7 +895,7 @@
function openScrapeModal() {
renderScrapeTree();
scrapeModal.hidden = false;
void loadChannels().then(renderScrapeTree);
void loadChannels(true).then(renderScrapeTree);
}
for (const [tab, id] of [
poc/src/media.ts
+69 -35
View File
@@ -425,63 +425,97 @@ export class MediaService {
const meta = this.engine.get(scrapeId);
const item = this.getMediaItem(scrapeId, itemId);
if (!meta || !item) return null;
if (meta.backend !== "slopbot" || !meta.guildId || !this.bot) return null;
return this.refreshUrls(meta.id, meta.channelId, item);
return this.refreshUrls(meta, item);
}
/** Re-fetch the source message and update expired URLs. Returns the fresh playable URL. */
private async refreshUrls(
scrapeId: string,
channelId: string,
item: MediaRow
): Promise<string | null> {
/**
* Re-fetch the source message and update expired URLs. Returns the fresh playable URL.
* Slopbot scrapes use the bot REST; DM and group-DM scrapes go through the core bridge.
*/
private async refreshUrls(meta: ScrapeMeta, item: MediaRow): Promise<string | null> {
try {
const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
let messageId: string | null = null;
try {
messageId = store.sourceMessageId(item.id);
} finally {
store.close();
}
if (!messageId || !this.bot) return null;
const channel = await this.bot.channels.fetch(channelId);
if (!channel?.isTextBased()) return null;
const message = await channel.messages.fetch(messageId);
if (!messageId) return null;
const path = (url: string) => url.split("?")[0];
const attachment = [...message.attachments.values()].find(
candidate => path(candidate.url) === path(item.url)
);
if (attachment) {
const update = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
const saveAndReturn = (update: { url?: string; playUrl?: string | null }, fresh: string) => {
const writer = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
try {
update.updateUrls(item.id, { url: attachment.url });
writer.updateUrls(item.id, update);
} finally {
update.close();
writer.close();
}
return fresh;
};
if (this.bot && meta.guildId && meta.backend === "slopbot") {
const channel = await this.bot.channels.fetch(meta.channelId);
if (channel?.isTextBased()) {
const message = await channel.messages.fetch(messageId);
const attachment = [...message.attachments.values()].find(
candidate => path(candidate.url) === path(item.url)
);
if (attachment) return saveAndReturn({ url: attachment.url }, attachment.url);
const embed = message.embeds.find(
candidate =>
(item.play_url &&
candidate.video?.url &&
path(candidate.video.url) === path(item.play_url)) ||
(candidate.video?.url && path(candidate.url ?? "") === path(item.url))
);
if (embed?.video?.url) {
return saveAndReturn(
{ playUrl: embed.video.url, url: embed.url ?? item.url },
embed.video.url
);
}
}
return attachment.url;
}
const embed = message.embeds.find(
candidate =>
(item.play_url && candidate.video?.url && path(candidate.video.url) === path(item.play_url)) ||
(candidate.video?.url && path(candidate.url ?? "") === path(item.url))
);
if (embed?.video?.url) {
const update = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
try {
update.updateUrls(item.id, { playUrl: embed.video.url, url: embed.url ?? item.url });
} finally {
update.close();
// The puppet path: the newest message below id+1 is exactly the target message.
const core = this.deps.getCore();
if (core && this.deps.isCoreConnected()) {
const result = (await core.fetchMessages(meta.channelId, 1, {
before: (BigInt(messageId) + 1n).toString()
})) as { messages?: Array<Record<string, any>> };
const message =
result?.messages?.find(candidate => candidate?.id === messageId) ??
result?.messages?.[0];
if (message) {
const attachment = (message.attachments ?? []).find(
(candidate: any) => candidate?.url && path(candidate.url) === path(item.url)
);
if (attachment?.url) return saveAndReturn({ url: attachment.url }, attachment.url);
const embed = (message.embeds ?? []).find(
(candidate: any) =>
(item.play_url &&
candidate?.videoUrl &&
path(candidate.videoUrl) === path(item.play_url)) ||
(candidate?.videoUrl && path(candidate.url ?? "") === path(item.url))
);
if (embed?.videoUrl) {
return saveAndReturn(
{ playUrl: embed.videoUrl, url: embed.url ?? item.url },
embed.videoUrl
);
}
}
return embed.video.url;
}
return null;
} catch (error) {
this.deps.log?.(`refresh failed for ${scrapeId}:${item.id}: ${String(error)}`);
this.deps.log?.(`refresh failed for ${meta.id}:${item.id}: ${String(error)}`);
return null;
}
}