zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cache embed thumbnails in download mode

Browse Source
This commit is contained in:
Fisto
2026-09-16 11:17:37 +02:00
parent 902646224f
commit 1fe204b91e
7 changed files with 161 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/engine.ts
+21 -8
View File
@@ -220,12 +220,24 @@ export class ScrapeEngine {
await this.sleep(this.gapMs());
}
if (meta.state === "done" && meta.mode === "download") {
await this.runDownloads(store, dir, meta);
}
if (meta.state === "done" && (meta.mode === "download" || this.options.previews !== false)) {
// Keep the job running until post-processing finishes, so a client never
// treats a half-archived scrape as complete.
meta.state = "running";
this.persist(dir, meta);
if (meta.state === "done" && this.options.previews !== false) {
await this.runPreviews(store, dir, meta);
if (meta.mode === "download") {
await this.runDownloads(store, dir, meta);
}
if (this.options.previews !== false && !this.stopRequested.has(meta.id)) {
await this.runPreviews(store, dir, meta);
}
if (meta.state === "running") {
meta.state = "done";
}
this.persist(dir, meta);
}
} catch (error) {
meta.state = "failed";
@@ -271,10 +283,11 @@ export class ScrapeEngine {
}
}
/** Cache small previews for Discord-hosted images. Runs in both modes. */
/** Cache small previews. In download mode this includes embed thumbnails. */
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const options = this.options.download ?? {};
const pending = store.pendingPreviews();
const embedPreviews = meta.mode === "download";
const options = { ...(this.options.download ?? {}), embedPreviews };
const pending = store.pendingPreviews(5000, embedPreviews);
for (const row of pending) {
if (this.stopRequested.has(meta.id)) {