zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add custom download destinations, confirmations, and the bot error feed

Browse Source
This commit is contained in:
Fisto
2026-09-16 11:53:59 +02:00
parent 973b9cff92
commit 42fd52b96b
14 changed files with 471 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/download.ts
+4 -2
View File
@@ -44,8 +44,10 @@ export interface DownloadOptions {
maxRetries?: number;
/** Preview size cap. Default 5 MiB. */
previewMaxBytes?: number;
/** Cache third-party embed thumbnails. Download mode only. */
/** Cache third-party embed thumbnails. */
embedPreviews?: boolean;
/** Absolute destination for downloaded files. Null or unset keeps them under the scrape dir. */
destinationDir?: string | null;
sleep?: (ms: number) => Promise<void>;
random?: () => number;
log?: (line: string) => void;
@@ -134,7 +136,7 @@ export async function downloadOne(
const file = `${hash}.${extension}`;
writeFileSync(join(dir, file), buffer);
return { id: media.id, state: "downloaded", path: join("media", file), hash, bytes: buffer.byteLength };
return { id: media.id, state: "downloaded", path: file, hash, bytes: buffer.byteLength };
}
} catch (error) {
lastError = error instanceof Error ? error.message : String(error);
media-scrape/src/engine.ts
+34 -7
View File
@@ -3,7 +3,8 @@
* Rules: docs/media-scraper.md.
*/
import { join } from "node:path";
import { copyFileSync, mkdirSync } from "node:fs";
import { dirname, join } from "node:path";
import {
downloadGapMs,
downloadOne,
@@ -58,6 +59,8 @@ export interface ScrapeEngineOptions {
previews?: boolean;
/** Read dimensions from a downloaded file. Defaults to ffprobe. */
probe?: (path: string) => Promise<{ width: number; height: number } | null>;
/** Called when a scrape fails. */
onError?: (scrapeId: string, message: string) => void;
}
interface QueuedJob {
@@ -91,6 +94,7 @@ export class ScrapeEngine {
guildId: string | null;
mode: ScrapeMode;
backend: ScrapeBackend;
downloadDir?: string | null;
}): Promise<ScrapeMeta> {
const now = this.now();
const id = uniqueScrapeId(this.options.rootDir, input.channelId, new Date(now));
@@ -111,7 +115,8 @@ export class ScrapeEngine {
messages: 0,
media: emptyMediaCounts(),
errors: 0,
lastError: null
lastError: null,
downloadDir: input.downloadDir ?? null
};
writeScrapeMeta(dir, meta);
@@ -251,6 +256,7 @@ export class ScrapeEngine {
meta.state = "failed";
meta.lastError = error instanceof Error ? error.message : String(error);
this.log(`${meta.id}: failed: ${meta.lastError}`);
this.options.onError?.(meta.id, meta.lastError);
} finally {
if ((meta.state === "done" || meta.state === "failed") && !meta.finishedAt) {
meta.finishedAt = new Date(this.now()).toISOString();
@@ -263,7 +269,13 @@ export class ScrapeEngine {
/** Fetch attachment bytes for a finished walk. Serial, hash-addressed, resumable. */
private async runDownloads(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const options = this.options.download ?? {};
const customDir = meta.downloadDir ?? null;
const options: DownloadOptions = {
...(this.options.download ?? {}),
destinationDir: customDir
};
const targetDir = customDir ? join(customDir, meta.id, "media") : join(dir, "media");
mkdirSync(targetDir, { recursive: true });
const pending = store.pendingDownloads();
for (const row of pending) {
@@ -272,21 +284,24 @@ export class ScrapeEngine {
return;
}
const outcome = await downloadOne(row, join(dir, "media"), {
const outcome = await downloadOne(row, targetDir, {
...options,
log: line => this.log(`${meta.id}: ${line}`)
});
if (outcome.state === "downloaded" && outcome.path && outcome.hash) {
const filePath = customDir ? join(targetDir, outcome.path) : join("media", outcome.path);
store.markDownloaded(row.id, {
path: outcome.path,
path: filePath,
hash: outcome.hash,
bytes: outcome.bytes ?? 0
});
meta.downloaded = (meta.downloaded ?? 0) + 1;
if (row.width === null || row.height === null) {
const dimensions = await (this.options.probe ?? probeDimensions)(join(dir, outcome.path));
const dimensions = await (this.options.probe ?? probeDimensions)(
join(targetDir, outcome.path)
);
if (dimensions) store.updateDimensions(row.id, dimensions.width, dimensions.height);
}
} else {
@@ -301,7 +316,11 @@ export class ScrapeEngine {
/** Cache small previews. Embed thumbnails are cached in both modes. */
private async runPreviews(store: ScrapeStore, dir: string, meta: ScrapeMeta): Promise<void> {
const options = { ...(this.options.download ?? {}), embedPreviews: true };
const options = {
...(this.options.download ?? {}),
embedPreviews: true,
destinationDir: meta.downloadDir ?? null
};
const pending = store.pendingPreviews(5000, true);
for (const row of pending) {
@@ -315,6 +334,14 @@ export class ScrapeEngine {
if (outcome.state === "downloaded" && outcome.hash) {
store.markPreview(row.id, { hash: outcome.hash, bytes: outcome.bytes ?? 0 });
meta.previewed = (meta.previewed ?? 0) + 1;
// Embed thumbnails also land in the custom destination, so the download has a
// viewable poster for embeds while the media tab keeps its local copy.
if (options.destinationDir && row.source === "embed") {
const target = join(options.destinationDir, meta.id, "thumbnails", `${outcome.hash}.webp`);
mkdirSync(dirname(target), { recursive: true });
copyFileSync(join(dir, "thumbs", `${outcome.hash}.webp`), target);
}
} else {
store.markPreviewError(row.id, outcome.error ?? outcome.state);
}
media-scrape/src/types.ts
+2
View File
@@ -94,6 +94,8 @@ export interface ScrapeMeta {
previewed?: number;
/** When the scrape reached done or failed. */
finishedAt?: string | null;
/** Absolute destination for downloaded files. Null means the local `media/` folder. */
downloadDir?: string | null;
}
export function emptyMediaCounts(): Record<MediaKind, number> {
media-scrape/test/run.ts
+73
View File
@@ -637,6 +637,79 @@ void (async () => {
assert.equal(existsSync(join(root, started.id, "thumbs", `${row.preview_hash}.webp`)), true);
});
await test("custom download destination keeps files off-archive and copies embed previews", async () => {
const dest = join(root, "custom-dest");
mkdirSync(dest, { recursive: true });
const backend = new FakeBackend([
{
messages: [
message("702", {
attachments: [attachment({ filename: "offsite.png", contentType: "image/png" })],
embeds: [
{
type: "video",
url: "https://www.youtube.com/watch?v=x",
title: null,
provider: "YouTube",
videoUrl: "https://www.youtube.com/embed/x",
imageUrl: null,
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
}
]
})
]
}
]);
const engine = new ScrapeEngine({
rootDir: root,
pageSize: 2,
pageMinMs: 0,
pageMaxMs: 0,
sleep: async () => {},
random: () => 0,
log: () => {},
previews: true,
download: {
fetchImpl: async () =>
new Response("bytes", { status: 200, headers: { "content-type": "image/png" } }),
minGapMs: 0,
maxGapMs: 0,
sleep: async () => {}
}
});
const started = await engine.start({
channelId: "130",
guildId: null,
mode: "download",
backend,
downloadDir: dest
});
await engine.processQueue();
const meta = engine.get(started.id);
assert.equal(meta?.downloadDir, dest);
assert.equal(meta?.downloaded, 1);
const store = new ScrapeStore(join(root, started.id));
const file = store.queryMedia({ kind: "image" })[0];
const filePath = file.file_path;
assert.ok(filePath && filePath.startsWith(dest));
assert.equal(existsSync(filePath), true);
assert.equal(existsSync(join(root, started.id, "media", filePath.split("/").pop() ?? "")), false);
const embed = store.queryMedia({ kind: "video" })[0];
assert.ok(embed.preview_hash);
assert.equal(
existsSync(join(dest, started.id, "thumbnails", `${embed.preview_hash}.webp`)),
true
);
assert.equal(existsSync(join(root, started.id, "thumbs", `${embed.preview_hash}.webp`)), true);
store.close();
});
await test("retention marks the oldest completed scrapes up to the limit", () => {
const scrapes = [
{ id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },