zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix recovery stale-URL handling and task accounting

Browse Source
This commit is contained in:
Fisto
2026-09-18 11:59:41 +02:00
parent 2ae90b23f4
commit e3edd9c200
6 changed files with 67 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CURRENT.md
+7
View File
@@ -40,6 +40,13 @@ mobile draft at 390px.
- [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB
cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`:
`previewed: 2`; thumbs served over `/api/media/<scrape>/<item>/thumb` (200).
- [x] C55 Recovery fix. The first recovery run parked live files because refreshed embed URLs
land in `play_url` while preview resolution kept using the stale `url`; the engine now resolves
embed previews from `play_url` first and re-resolves from the refreshed URL on retry, and both
refresh branches canonicalize `url` to the direct media link. The task also counts rows that
park as suspects as `deferred` (not gone), tries each row once per run instead of looping, and
initializes the slopbot so guild scrapes refresh through the bot. The 192 suspects from the
broken run were converted back to unverified so they are retried.
- [x] C54 Message links, suspect retries, and preview recovery. Items now carry `messageUrl`
(from `media_sources` + scrape guild), the viewer shows/copies it instead of the expiring CDN
link and opens a fresh stream for View full, YouTube items offer an **Open on YouTube**
docs/media-scraper.md
+4
View File
@@ -217,6 +217,10 @@ The archive never treats a transient HTTP answer as a dead link on the first try
- A preview or download that fails with an HTTP status on Discord's CDN is retried once with a
**refreshed URL** from the source message (the signed link expires in about a day). Only if the
refreshed fetch fails is the row parked.
- Refreshing an embed canonicalizes both links: `play_url` gets the provider's direct media URL and
`url` falls back to it when Discord's `embed.url` is missing. Preview resolution for embeds uses
`play_url` first, so a refreshed file is never fetched through an expired `url`. This is what the
recovery task relies on.
- Parked rows become **suspects**: they get one delayed retry (about 50 s). The retry runs after
the batch, not immediately. If a batch is two-thirds suspect failures, the cooldown doubles
(floor 60 s) and the retries run before the next batch.
media-scrape/src/engine.ts
+8 -2
View File
@@ -709,7 +709,8 @@ export class ScrapeEngine {
// resolve one from the provider or the page before caching it.
let previewRow = row;
if (row.source === "embed" && !row.preview_url) {
const resolved = await resolveEmbedPreview(row, options);
// play_url is the refreshed direct media link; url may carry an expired signature.
const resolved = await resolveEmbedPreview({ ...row, url: row.play_url ?? row.url }, options);
if (!resolved) {
store.markPreviewError(row.id, "no preview source");
result.skipped++;
@@ -812,7 +813,12 @@ export class ScrapeEngine {
playUrl: row.play_url
});
if (refreshed && refreshed !== previewRow.url) {
const retry = await downloadPreview({ ...previewRow, url: refreshed }, join(dir, "thumbs"), options);
let retryRow = { ...previewRow, url: refreshed };
if (row.source === "embed") {
const again = await resolveEmbedPreview({ ...row, url: refreshed }, options);
if (again) retryRow = { ...retryRow, preview_url: again };
}
const retry = await downloadPreview(retryRow, join(dir, "thumbs"), options);
if (retry.state === "downloaded" && retry.hash) {
store.markPreview(row.id, {
hash: retry.hash,
media-scrape/src/store.ts
+13 -4
View File
@@ -572,13 +572,18 @@ export class ScrapeStore {
this.db.prepare("UPDATE media SET preview_suspect = NULL, preview_retry_at = NULL WHERE id = ?").run(id);
}
/** Discord-CDN rows closed out with an HTTP error; the recovery task retries these. */
/**
* Discord-CDN rows the recovery task retries: closed out with an HTTP error, or suspect rows
* whose single retry already failed (they may have been hit by the same stale-URL bug).
*/
discordPreviewFailures(limit = 5000): MediaRow[] {
return this.db
.prepare(
`SELECT * FROM media
WHERE preview_hash IS NULL AND preview_error IS NOT NULL
AND lower(url) LIKE '%discordapp%'
WHERE preview_hash IS NULL
AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
AND (preview_error IS NOT NULL
OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))
ORDER BY id LIMIT ?`
)
.all(limit) as unknown as MediaRow[];
@@ -591,7 +596,11 @@ export class ScrapeStore {
discordPreviewFailureCount(): number {
const row = this.db
.prepare(
"SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND preview_error IS NOT NULL AND lower(url) LIKE '%discordapp%'"
`SELECT COUNT(*) AS count FROM media
WHERE preview_hash IS NULL
AND lower(COALESCE(url, '') || ' ' || COALESCE(play_url, '')) LIKE '%discordapp%'
AND (preview_error IS NOT NULL
OR (preview_suspect IS NOT NULL AND preview_retry_at IS NULL))`
)
.get() as { count: number | null } | undefined;
return row?.count ?? 0;
poc/public/media.js
+1
View File
@@ -911,6 +911,7 @@
const parts = [`${status.total} to retry`];
if (status.processed) parts.push(`${status.processed} processed`);
if (status.recovered) parts.push(`${status.recovered} recovered`);
if (status.deferred) parts.push(`${status.deferred} deferred`);
if (status.gone) parts.push(`${status.gone} unavailable`);
if (status.currentScrapeId) parts.push(`on ${channelLabel(status.currentScrapeId)}`);
head.append(el("span", "vk-bot-job-meta", parts.join(" \u00b7 ")));
poc/src/media.ts
+34 -7
View File
@@ -194,6 +194,7 @@ export class MediaService {
state: "idle" as "idle" | "running" | "paused" | "done",
processed: 0,
recovered: 0,
deferred: 0,
gone: 0,
startedAt: null as number | null,
finishedAt: null as number | null,
@@ -327,6 +328,7 @@ export class MediaService {
total: number;
processed: number;
recovered: number;
deferred: number;
gone: number;
startedAt: string | null;
finishedAt: string | null;
@@ -353,6 +355,7 @@ export class MediaService {
total,
processed: this.recover.processed,
recovered: this.recover.recovered,
deferred: this.recover.deferred,
gone: this.recover.gone,
startedAt: this.recover.startedAt ? new Date(this.recover.startedAt).toISOString() : null,
finishedAt: this.recover.finishedAt ? new Date(this.recover.finishedAt).toISOString() : null,
@@ -367,8 +370,15 @@ export class MediaService {
this.recover.startedAt = this.recover.startedAt ?? Date.now();
this.recover.processed = 0;
this.recover.recovered = 0;
this.recover.deferred = 0;
this.recover.gone = 0;
this.deps.log?.("recover: retrying Discord-CDN previews marked gone");
// Guild scrapes refresh fastest through the bot REST; without it the puppet is tried.
if (this.engine.list().some(meta => meta.backend === "slopbot")) {
void this.ensureSlopbot().then(ok => {
if (!ok) this.deps.log?.("recover: slopbot unavailable, falling back to the puppet");
});
}
void this.runRecover();
}
@@ -389,6 +399,10 @@ export class MediaService {
.filter(meta => meta.state !== "deleted" && meta.state !== "deleting")
.map(meta => meta.id);
// Each row is tried once per run; rows whose URL cannot be refreshed keep their state
// instead of being re-processed forever.
const attempted = new Set<string>();
while (this.recover.state === "running") {
let acted = false;
@@ -406,7 +420,10 @@ export class MediaService {
}
try {
const rows = store.discordPreviewFailures(10);
const rows = store
.discordPreviewFailures(200)
.filter(row => !attempted.has(`${scrapeId}:${row.id}`))
.slice(0, 10);
if (!rows.length) continue;
acted = true;
@@ -415,6 +432,9 @@ export class MediaService {
for (const row of rows) {
if (this.recover.state !== "running") break;
attempted.add(`${scrapeId}:${row.id}`);
this.recover.processed++;
const fresh = await this.refreshUrls(meta, {
...row,
scrapeId,
@@ -422,17 +442,19 @@ export class MediaService {
} as MediaRow & { scrapeId: string; channelId: string });
if (fresh) {
store.clearPreviewError(row.id);
store.clearSuspect(row.id);
cleared.push(row);
} else {
this.recover.gone++;
}
this.recover.processed++;
}
if (cleared.length && this.recover.state === "running") {
await this.engine.previewBatch(scrapeId, cleared.length);
for (const row of cleared) {
if (store.getMedia(row.id)?.preview_hash) this.recover.recovered++;
const after = store.getMedia(row.id);
if (after?.preview_hash) this.recover.recovered++;
else if (after?.preview_suspect) this.recover.deferred++;
else this.recover.gone++;
}
}
@@ -452,11 +474,14 @@ export class MediaService {
this.taskHistory.unshift({
name: "Recover Discord previews",
at: new Date().toISOString(),
detail: `${this.recover.recovered} recovered, ${this.recover.gone} unavailable`
detail:
`${this.recover.recovered} recovered, ` +
`${this.recover.deferred} deferred, ${this.recover.gone} unavailable`
});
if (this.taskHistory.length > 10) this.taskHistory.length = 10;
this.deps.log?.(
`recover: done, ${this.recover.recovered} recovered, ${this.recover.gone} unavailable`
`recover: done, ${this.recover.recovered} recovered, ` +
`${this.recover.deferred} deferred, ${this.recover.gone} unavailable`
);
}
@@ -767,8 +792,9 @@ export class MediaService {
(candidate.video?.url && path(candidate.url ?? "") === path(item.url))
);
if (embed?.video?.url) {
// The direct media link is always fresh; a bare embed.url can be stale.
return saveAndReturn(
{ playUrl: embed.video.url, url: embed.url ?? item.url },
{ playUrl: embed.video.url, url: embed.url ?? embed.video.url },
embed.video.url
);
}
@@ -799,8 +825,9 @@ export class MediaService {
(candidate?.videoUrl && path(candidate.url ?? "") === path(item.url))
);
if (embed?.videoUrl) {
// Same as the bot path: keep the canonical url fresh too.
return saveAndReturn(
{ playUrl: embed.videoUrl, url: embed.url ?? item.url },
{ playUrl: embed.videoUrl, url: embed.url ?? embed.videoUrl },
embed.videoUrl
);
}