zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add custom download destinations, confirmations, and the bot error feed

Browse Source
This commit is contained in:
Fisto
2026-09-16 11:53:59 +02:00
parent 973b9cff92
commit 42fd52b96b
14 changed files with 471 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/download.ts
+4 -2
View File
@@ -44,8 +44,10 @@ export interface DownloadOptions {
maxRetries?: number;
/** Preview size cap. Default 5 MiB. */
previewMaxBytes?: number;
/** Cache third-party embed thumbnails. Download mode only. */
/** Cache third-party embed thumbnails. */
embedPreviews?: boolean;
/** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */
destinationDir?: string | null;
sleep?: (ms: number) => Promise<void>;
random?: () => number;
log?: (line: string) => void;
@@ -134,7 +136,7 @@ export async function downloadOne(
const file = `${hash}.${extension}`;
writeFileSync(join(dir, file), buffer);
return { id: media.id, state: "downloaded", path: join("media", file), hash, bytes: buffer.byteLength };
return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength };
}
} catch (error) {
lastError = error instanceof Error ? error.message : String(error);
media-scrape/src/engine.ts
+34 -7
View File
@@ -3,7 +3,8 @@
* Rules: docs/media-scraper.md.
*/
import { join } from "node:path";
import { copyFileSync, mkdirSync } from "node:fs";
import { dirname, join } from "node:path";
import {
downloadGapMs,
downloadOne,
@@ -58,6 +59,8 @@ export interface ScrapeEngineOptions {
previews?: boolean;
/** Read dimensions from a downloaded file. Defaults to ffprobe. */
probe?: (path: string) => Promise<{ width: number; height: number } | null>;
/** Called when a scrape fails. */
onError?: (scrapeId: string, message: string) => void;
}
interface QueuedJob {
@@ -91,6 +94,7 @@ export class ScrapeEngine {
guildId: string | null;
mode: ScrapeMode;
backend: ScrapeBackend;
downloadDir?: string | null;
}): Promise<ScrapeMeta> {
const now = this.now();
const id = uniqueScrapeId(this.options.rootDir, input.channelId, new Date(now));
@@ -111,7 +115,8 @@ export class ScrapeEngine {
messages: 0,
media: emptyMediaCounts(),
errors: 0,
lastError: null
lastError: null,
downloadDir: input.downloadDir ?? null
};
writeScrapeMeta(dir, meta);
@@ -251,6 +256,7 @@ export class ScrapeEngine {
meta.state = "failed";
meta.lastError = error instanceof Error ? error.message : String(error);
this.log(`${meta.id}: failed: ${meta.lastError}`);
this.options.onError?.(meta.id, meta.lastError);
} finally {
if ((meta.state === "done" || meta.state === "failed") && !meta.finishedAt) {
meta.finishedAt = new Date(this.now()).toISOString();
@@ -263,7 +269,13 @@ export class ScrapeEngine {
/** Fetch attachment bytes for a finished walk. Serial, hash-addressed, resumable. */
private async runDownloads(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const options = this.options.download ?? {};
const customDir = meta.downloadDir ?? null;
const options: DownloadOptions = {
...(this.options.download ?? {}),
destinationDir: customDir
};
const targetDir = customDir ? join(customDir, meta.id, "media") : join(dir, "media");
mkdirSync(targetDir, { recursive: true });
const pending = store.pendingDownloads();
for (const row of pending) {
@@ -272,21 +284,24 @@ export class ScrapeEngine {
return;
}
const outcome = await downloadOne(row, join(dir, "media"), {
const outcome = await downloadOne(row, targetDir, {
...options,
log: line => this.log(`${meta.id}: ${line}`)
});
if (outcome.state === "downloaded" && outcome.path && outcome.hash) {
const filePath = customDir ? join(targetDir, outcome.path) : join("media", outcome.path);
store.markDownloaded(row.id, {
path: outcome.path,
path: filePath,
hash: outcome.hash,
bytes: outcome.bytes ?? 0
});
meta.downloaded = (meta.downloaded ?? 0) + 1;
if (row.width === null || row.height === null) {
const dimensions = await (this.options.probe ?? probeDimensions)(join(dir, outcome.path));
const dimensions = await (this.options.probe ?? probeDimensions)(
join(targetDir, outcome.path)
);
if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height);
}
} else {
@@ -301,7 +316,11 @@ export class ScrapeEngine {
/** Cache small previews. Embed thumbnails are cached in both modes. */
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const options = { ...(this.options.download ?? {}), embedPreviews: true };
const options = {
...(this.options.download ?? {}),
embedPreviews: true,
destinationDir: meta.downloadDir ?? null
};
const pending = store.pendingPreviews(5000, true);
for (const row of pending) {
@@ -315,6 +334,14 @@ export class ScrapeEngine {
if (outcome.state === "downloaded" && outcome.hash) {
store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0 });
meta.previewed = (meta.previewed ?? 0) + 1;
// Embed thumbnails also land in the custom destination, so the download has a
// viewable poster for embeds while the media tab keeps its local copy.
if (options.destinationDir && row.source === "embed") {
const target = join(options.destinationDir, meta.id, "thumbnails", `${outcome.hash}.webp`);
mkdirSync(dirname(target), { recursive: true });
copyFileSync(join(dir, "thumbs", `${outcome.hash}.webp`), target);
}
} else {
store.markPreviewError(row.id, outcome.error ?? outcome.state);
}
media-scrape/src/types.ts
+2
View File
@@ -94,6 +94,8 @@ export interface ScrapeMeta {
previewed?: number;
/** When the scrape reached done or failed. */
finishedAt?: string | null;
/** Absolute destination for downloaded files. Null means the local `media/` folder. */
downloadDir?: string | null;
}
export function emptyMediaCounts(): Record<MediaKind, number> {