1306 lines
48 KiB
TypeScript
1306 lines
48 KiB
TypeScript
/*
|
|
* Media archive service: the daemon side of media-scrape/. See docs/media.md.
|
|
*
|
|
* Owns the scrape engine, the slopbot client, and the read path the Web UI and
|
|
* (later) MCP tools call.
|
|
*/
|
|
|
|
import { rmSync } from "node:fs";
|
|
import { isAbsolute, join } from "node:path";
|
|
import {
|
|
BackfillQueue,
|
|
betterCandidate,
|
|
messageUrlFor,
|
|
sortCandidates,
|
|
dirSizeBytes,
|
|
isNearLimit,
|
|
planRetention,
|
|
ScrapeEngine,
|
|
ScrapeStore,
|
|
scrapeDir,
|
|
TitleLane,
|
|
type TitleLaneStatus,
|
|
snowflakeFromIso,
|
|
resolveDbRoot,
|
|
type MediaKind,
|
|
type MediaRow,
|
|
PROVIDER_FILTERS,
|
|
type BackfillSettings,
|
|
type BackfillStatus,
|
|
type MediaSort,
|
|
type ScrapeBackend,
|
|
type ScrapeMeta
|
|
} from "../../media-scrape/src/index.js";
|
|
import { PuppetBackend } from "../../media-scrape/src/puppet-backend.js";
|
|
import {
|
|
countMessages,
|
|
loadSlopbotConfig,
|
|
slopbotFetcher,
|
|
slopbotStatus,
|
|
SlopbotBackend,
|
|
startSlopbot,
|
|
type SlopbotFetcher,
|
|
type SlopbotRest,
|
|
type SlopbotStatus
|
|
} from "../../media-scrape/slopbot/index.js";
|
|
import type { CoreClient } from "./core-client.js";
|
|
import type { MediaPreferences } from "./preferences.js";
|
|
|
|
type BotClient = Awaited<ReturnType<typeof startSlopbot>>;
|
|
|
|
export interface MediaServiceDeps {
|
|
getCore: () => CoreClient | undefined;
|
|
isCoreConnected: () => boolean;
|
|
getMediaPrefs: () => MediaPreferences;
|
|
log?: (line: string) => void;
|
|
}
|
|
|
|
export interface StartScrapeInput {
|
|
channelId: string;
|
|
guildId?: string | null;
|
|
mode?: "links" | "download";
|
|
backend?: "auto" | "slopbot" | "puppet";
|
|
/** Set by a re-run of the command after the near-limit warning. */
|
|
confirm?: boolean;
|
|
/** Absolute destination for downloaded files. Null keeps them under the scrape dir. */
|
|
downloadDir?: string | null;
|
|
/** From-date as ISO; the walk stops below it. */
|
|
after?: string | null;
|
|
/** To-date as ISO; the walk starts just below it. */
|
|
before?: string | null;
|
|
}
|
|
|
|
export interface MediaErrorEntry {
|
|
at: string;
|
|
source: string;
|
|
message: string;
|
|
}
|
|
|
|
export interface ScrapeBytes {
|
|
id: string;
|
|
channelId: string;
|
|
state: string;
|
|
createdAt: string;
|
|
bytes: number;
|
|
}
|
|
|
|
export interface MediaStorageStatus {
|
|
usedBytes: number;
|
|
limitBytes: number;
|
|
warnPercent: number;
|
|
overrunPercent: number;
|
|
nearLimit: boolean;
|
|
byKind: Record<MediaKind, number>;
|
|
scrapes: ScrapeBytes[];
|
|
}
|
|
|
|
/** Thrown when a scrape starts near the storage limit without confirmation. */
|
|
export class MediaLimitError extends Error {
|
|
readonly code = "MEDIA_LIMIT";
|
|
|
|
constructor(
|
|
message: string,
|
|
readonly usedBytes: number,
|
|
readonly limitBytes: number
|
|
) {
|
|
super(message);
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes: number): string {
|
|
const gib = bytes / 1024 ** 3;
|
|
return gib >= 1 ? `${gib.toFixed(2)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`;
|
|
}
|
|
|
|
export interface MediaItemResult {
|
|
scrapeId: string;
|
|
itemId: number;
|
|
channelId: string;
|
|
guildId: string | null;
|
|
/** The message that carried the media; the permanent link target. */
|
|
messageId: string | null;
|
|
messageUrl: string | null;
|
|
kind: MediaKind;
|
|
source: string | null;
|
|
authorId: string | null;
|
|
authorName: string | null;
|
|
postedAt: string | null;
|
|
width: number | null;
|
|
height: number | null;
|
|
aspectRatio: number | null;
|
|
contentType: string | null;
|
|
durationSecs: number | null;
|
|
url: string;
|
|
playUrl: string | null;
|
|
/** The remote thumbnail URL from Discord's embed data (may be an animated GIF). */
|
|
remotePreviewUrl: string | null;
|
|
/** Why no preview exists, when a fetch failed (for example "http 404"). */
|
|
previewError: string | null;
|
|
/** Why no title was resolved (provider oEmbed and Discord's embed both failed). */
|
|
titleError: string | null;
|
|
/** Local animated GIF for hover playback, when the original was cached. */
|
|
animUrl: string | null;
|
|
localUrl: string | null;
|
|
thumbUrl: string | null;
|
|
provider: string | null;
|
|
downloaded: boolean;
|
|
/** Embed title (YouTube and friends), when Discord provided one. */
|
|
title: string | null;
|
|
/** YouTube channel name kept from oEmbed, even if the video later disappears. */
|
|
titleAuthor: string | null;
|
|
createdAt: string;
|
|
}
|
|
|
|
const FILE_CONTENT_TYPES: Record<string, string> = {
|
|
png: "image/png",
|
|
jpg: "image/jpeg",
|
|
jpeg: "image/jpeg",
|
|
gif: "image/gif",
|
|
webp: "image/webp",
|
|
avif: "image/avif",
|
|
mp4: "video/mp4",
|
|
webm: "video/webm",
|
|
mov: "video/quicktime",
|
|
mkv: "video/x-matroska",
|
|
mp3: "audio/mpeg",
|
|
ogg: "audio/ogg",
|
|
wav: "audio/wav",
|
|
m4a: "audio/mp4",
|
|
flac: "audio/flac"
|
|
};
|
|
|
|
export interface MediaStatus {
|
|
slopbot: SlopbotStatus;
|
|
scrapes: ScrapeMeta[];
|
|
/** ISO time while a Discord rate limit holds media traffic. */
|
|
limitedUntil: string | null;
|
|
/** The common providers, for the "Do not include" shortcuts in the Advanced drawer. */
|
|
providerFilters: string[];
|
|
}
|
|
|
|
export class MediaService {
|
|
private readonly rootDir = resolveDbRoot();
|
|
private readonly engine: ScrapeEngine;
|
|
private readonly backfill: BackfillQueue;
|
|
private readonly titles: TitleLane;
|
|
|
|
private bot: BotClient | null = null;
|
|
private botFetcher: SlopbotFetcher | null = null;
|
|
private botStarting: Promise<void> | null = null;
|
|
|
|
/** The newest bot and scrape errors, capped. Surfaced in the Bot tab. */
|
|
private readonly errors: MediaErrorEntry[] = [];
|
|
|
|
/** One-shot maintenance: re-fetch previews for Discord rows closed out with HTTP errors. */
|
|
private readonly recover = {
|
|
state: "idle" as "idle" | "running" | "paused" | "done",
|
|
processed: 0,
|
|
recovered: 0,
|
|
deferred: 0,
|
|
gone: 0,
|
|
startedAt: null as number | null,
|
|
finishedAt: null as number | null,
|
|
currentScrapeId: null as string | null
|
|
};
|
|
/** Finished maintenance tasks, newest first. */
|
|
private readonly taskHistory: Array<{ name: string; at: string; detail: string }> = [];
|
|
|
|
constructor(private readonly deps: MediaServiceDeps) {
|
|
const tunables = this.deps.getMediaPrefs().tunables;
|
|
|
|
this.engine = new ScrapeEngine({
|
|
rootDir: this.rootDir,
|
|
log: line => this.deps.log?.(line),
|
|
onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`),
|
|
walkCyclePages: tunables.scrape.walkCyclePages,
|
|
rateLimitPauseMs: tunables.timers.rateLimitPauseMs,
|
|
pageMinMs: tunables.timers.pageMinMs,
|
|
pageMaxMs: tunables.timers.pageMaxMs,
|
|
maxRetries: tunables.scrape.maxRetries,
|
|
previewMaxAttempts: tunables.backfill.maxAttempts,
|
|
suspectRetryMs: 50_000,
|
|
embedMeta: async (scrapeId, media) => this.embedMetaFor(scrapeId, media),
|
|
refreshUrl: async (scrapeId, media) => {
|
|
const meta = this.engine.get(scrapeId);
|
|
if (!meta) return null;
|
|
const item = this.getMediaItem(scrapeId, media.id);
|
|
if (!item) return null;
|
|
return this.refreshUrls(meta, item);
|
|
},
|
|
download: {
|
|
maxFileBytes: tunables.scrape.maxFileBytes,
|
|
previewMaxBytes: tunables.backfill.maxBytes,
|
|
previewMinGapMs: tunables.backfill.previewMinGapMs,
|
|
previewMaxGapMs: tunables.backfill.previewMaxGapMs,
|
|
previewBatchSize: tunables.backfill.batchSize,
|
|
previewBatchPauseMs: tunables.backfill.batchPauseMs,
|
|
minGapMs: tunables.scrape.downloadMinGapMs,
|
|
maxGapMs: tunables.scrape.downloadMaxGapMs
|
|
}
|
|
});
|
|
|
|
this.backfill = new BackfillQueue(this.engine, {
|
|
rootDir: this.rootDir,
|
|
getSettings: () => this.backfillSettings(),
|
|
getGaps: () => {
|
|
const live = this.deps.getMediaPrefs().tunables;
|
|
return { minGapMs: live.backfill.previewMinGapMs, maxGapMs: live.backfill.previewMaxGapMs };
|
|
},
|
|
log: line => this.deps.log?.(line)
|
|
});
|
|
|
|
// The title catch-up lane talks to YouTube only, so it runs beside the backfill queue.
|
|
// It auto-starts because it is a one-time catch-up: with nothing pending it exits at once.
|
|
this.titles = new TitleLane(this.engine, {
|
|
rootDir: this.rootDir,
|
|
log: line => this.deps.log?.(line)
|
|
});
|
|
this.titles.start();
|
|
|
|
const stale = this.engine.reconcileStaleJobs();
|
|
if (stale.length) {
|
|
this.deps.log?.(`paused ${stale.length} stale scrape(s) left running by a previous daemon`);
|
|
}
|
|
|
|
// Preview passes only run when a walk ends; the queue fills in scrapes that paused early.
|
|
this.backfill.autoStart();
|
|
}
|
|
|
|
private backfillSettings(): BackfillSettings {
|
|
const tunables = this.deps.getMediaPrefs().tunables;
|
|
return {
|
|
batchSize: tunables.backfill.batchSize,
|
|
batchPauseMs: tunables.backfill.batchPauseMs,
|
|
auto: tunables.backfill.auto
|
|
};
|
|
}
|
|
|
|
/** Push preference changes into the live engine and backfill queue. */
|
|
applyTunables(): void {
|
|
const tunables = this.deps.getMediaPrefs().tunables;
|
|
this.engine.updateOptions({
|
|
walkCyclePages: tunables.scrape.walkCyclePages,
|
|
rateLimitPauseMs: tunables.timers.rateLimitPauseMs,
|
|
pageMinMs: tunables.timers.pageMinMs,
|
|
pageMaxMs: tunables.timers.pageMaxMs,
|
|
maxRetries: tunables.scrape.maxRetries,
|
|
previewMaxAttempts: tunables.backfill.maxAttempts,
|
|
download: {
|
|
maxFileBytes: tunables.scrape.maxFileBytes,
|
|
previewMaxBytes: tunables.backfill.maxBytes,
|
|
previewMinGapMs: tunables.backfill.previewMinGapMs,
|
|
previewMaxGapMs: tunables.backfill.previewMaxGapMs,
|
|
previewBatchSize: tunables.backfill.batchSize,
|
|
previewBatchPauseMs: tunables.backfill.batchPauseMs,
|
|
minGapMs: tunables.scrape.downloadMinGapMs,
|
|
maxGapMs: tunables.scrape.downloadMaxGapMs
|
|
}
|
|
});
|
|
}
|
|
|
|
/** Backfill jobs, totals, and pacing, plus the engine's live cooldown. */
|
|
backfillStatus(): BackfillStatus {
|
|
const status = this.backfill.list();
|
|
const limit = this.engine.rateLimitStatus();
|
|
|
|
return {
|
|
...status,
|
|
limitedUntil: limit.active
|
|
? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString()
|
|
: status.limitedUntil
|
|
};
|
|
}
|
|
|
|
titleStatus(): TitleLaneStatus {
|
|
return this.titles.status();
|
|
}
|
|
|
|
startTitles(): TitleLaneStatus {
|
|
this.titles.start();
|
|
return this.titles.status();
|
|
}
|
|
|
|
pauseTitles(): TitleLaneStatus {
|
|
this.titles.pause();
|
|
return this.titles.status();
|
|
}
|
|
|
|
/** The recovery task with its live remaining count and the task history. */
|
|
recoverStatus(): {
|
|
state: string;
|
|
total: number;
|
|
processed: number;
|
|
recovered: number;
|
|
deferred: number;
|
|
gone: number;
|
|
startedAt: string | null;
|
|
finishedAt: string | null;
|
|
currentScrapeId: string | null;
|
|
history: Array<{ name: string; at: string; detail: string }>;
|
|
} {
|
|
let total = 0;
|
|
for (const meta of this.engine.list()) {
|
|
if (meta.state === "deleted" || meta.state === "deleting") continue;
|
|
try {
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
total += store.discordPreviewFailureCount();
|
|
} finally {
|
|
store.close();
|
|
}
|
|
} catch {
|
|
// an unreadable index has nothing to recover
|
|
}
|
|
}
|
|
|
|
return {
|
|
state: this.recover.state,
|
|
total,
|
|
processed: this.recover.processed,
|
|
recovered: this.recover.recovered,
|
|
deferred: this.recover.deferred,
|
|
gone: this.recover.gone,
|
|
startedAt: this.recover.startedAt ? new Date(this.recover.startedAt).toISOString() : null,
|
|
finishedAt: this.recover.finishedAt ? new Date(this.recover.finishedAt).toISOString() : null,
|
|
currentScrapeId: this.recover.currentScrapeId,
|
|
history: [...this.taskHistory]
|
|
};
|
|
}
|
|
|
|
startRecover(): void {
|
|
if (this.recover.state === "running") return;
|
|
this.recover.state = "running";
|
|
this.recover.startedAt = this.recover.startedAt ?? Date.now();
|
|
this.recover.processed = 0;
|
|
this.recover.recovered = 0;
|
|
this.recover.deferred = 0;
|
|
this.recover.gone = 0;
|
|
this.deps.log?.("recover: retrying Discord-CDN previews marked gone");
|
|
// Guild scrapes refresh fastest through the bot REST; without it the puppet is tried.
|
|
if (this.engine.list().some(meta => meta.backend === "slopbot")) {
|
|
void this.ensureSlopbot().then(ok => {
|
|
if (!ok) this.deps.log?.("recover: slopbot unavailable, falling back to the puppet");
|
|
});
|
|
}
|
|
void this.runRecover();
|
|
}
|
|
|
|
pauseRecover(): void {
|
|
if (this.recover.state !== "running") return;
|
|
this.recover.state = "paused";
|
|
this.deps.log?.("recover: pausing");
|
|
}
|
|
|
|
/**
|
|
* Walk every scrape's Discord-CDN preview failures, refresh the signed URL from the source
|
|
* message (slopbot REST for guild scrapes, the puppet for DMs and group chats), then let the
|
|
* engine fetch the preview. Rows whose URL cannot be refreshed are counted as gone.
|
|
*/
|
|
private async runRecover(): Promise<void> {
|
|
const scrapeIds = this.engine
|
|
.list()
|
|
.filter(meta => meta.state !== "deleted" && meta.state !== "deleting")
|
|
.map(meta => meta.id);
|
|
|
|
// Each row is tried once per run; rows whose URL cannot be refreshed keep their state
|
|
// instead of being re-processed forever.
|
|
const attempted = new Set<string>();
|
|
|
|
while (this.recover.state === "running") {
|
|
let acted = false;
|
|
|
|
for (const scrapeId of scrapeIds) {
|
|
if (this.recover.state !== "running") break;
|
|
|
|
const meta = this.engine.get(scrapeId);
|
|
if (!meta) continue;
|
|
|
|
let store: ScrapeStore | null = null;
|
|
try {
|
|
store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
const rows = store
|
|
.discordPreviewFailures(200)
|
|
.filter(row => !attempted.has(`${scrapeId}:${row.id}`))
|
|
.slice(0, 10);
|
|
if (!rows.length) continue;
|
|
|
|
acted = true;
|
|
this.recover.currentScrapeId = scrapeId;
|
|
const cleared: MediaRow[] = [];
|
|
|
|
for (const row of rows) {
|
|
if (this.recover.state !== "running") break;
|
|
attempted.add(`${scrapeId}:${row.id}`);
|
|
this.recover.processed++;
|
|
|
|
const fresh = await this.refreshUrls(meta, {
|
|
...row,
|
|
scrapeId,
|
|
channelId: meta.channelId
|
|
} as MediaRow & { scrapeId: string; channelId: string });
|
|
if (fresh) {
|
|
store.clearPreviewError(row.id);
|
|
store.clearSuspect(row.id);
|
|
cleared.push(row);
|
|
} else {
|
|
// No fresh URL after a full run: unavailable. Mark it so later runs skip it.
|
|
store.markUnavailableAfterRecovery(row.id);
|
|
this.recover.gone++;
|
|
}
|
|
}
|
|
|
|
if (cleared.length && this.recover.state === "running") {
|
|
await this.engine.previewBatch(scrapeId, cleared.length);
|
|
for (const row of cleared) {
|
|
const after = store.getMedia(row.id);
|
|
if (after?.preview_hash) {
|
|
this.recover.recovered++;
|
|
} else {
|
|
// Refreshed URL plus a fetch still failed: the file is gone. Mark it
|
|
// so later recovery runs do not retry it again.
|
|
if (after?.preview_suspect) store.markUnavailableAfterRecovery(row.id);
|
|
this.recover.gone++;
|
|
}
|
|
}
|
|
}
|
|
} finally {
|
|
store?.close();
|
|
}
|
|
|
|
if (acted) break;
|
|
}
|
|
|
|
if (!acted) break;
|
|
}
|
|
|
|
if (this.recover.state === "running") this.recover.state = "done";
|
|
this.recover.finishedAt = Date.now();
|
|
this.recover.currentScrapeId = null;
|
|
this.taskHistory.unshift({
|
|
name: "Recover Discord previews",
|
|
at: new Date().toISOString(),
|
|
detail:
|
|
`${this.recover.recovered} recovered, ` +
|
|
`${this.recover.deferred} deferred, ${this.recover.gone} unavailable`
|
|
});
|
|
if (this.taskHistory.length > 10) this.taskHistory.length = 10;
|
|
this.deps.log?.(
|
|
`recover: done, ${this.recover.recovered} recovered, ` +
|
|
`${this.recover.deferred} deferred, ${this.recover.gone} unavailable`
|
|
);
|
|
}
|
|
|
|
startBackfill(chatId?: string): BackfillStatus {
|
|
this.backfill.start(chatId);
|
|
return this.backfillStatus();
|
|
}
|
|
|
|
pauseBackfill(chatId?: string): BackfillStatus {
|
|
this.backfill.pause(chatId);
|
|
return this.backfillStatus();
|
|
}
|
|
|
|
/** Log slopbot in on first use. False when it is disabled or the login failed. */
|
|
private async ensureSlopbot(): Promise<boolean> {
|
|
if (this.bot) return true;
|
|
|
|
const config = loadSlopbotConfig();
|
|
if (!config.enabled) return false;
|
|
|
|
if (!this.botStarting) {
|
|
this.botStarting = startSlopbot(config)
|
|
.then(client => {
|
|
this.bot = client;
|
|
this.botFetcher = slopbotFetcher(client);
|
|
this.deps.log?.("slopbot connected");
|
|
})
|
|
.catch(error => {
|
|
const message = `slopbot login failed: ${String(error)}`;
|
|
this.deps.log?.(message);
|
|
this.recordError("slopbot", message);
|
|
});
|
|
}
|
|
|
|
await this.botStarting;
|
|
return this.bot !== null;
|
|
}
|
|
|
|
private puppetBackend(): ScrapeBackend {
|
|
return new PuppetBackend(async (channelId, limit, options) => {
|
|
const core = this.deps.getCore();
|
|
if (!core || !this.deps.isCoreConnected()) {
|
|
throw new Error("the core bridge is not connected");
|
|
}
|
|
return (await core.fetchMessages(channelId, limit, options)) as { messages?: unknown[] };
|
|
});
|
|
}
|
|
|
|
/** Combined size of every scrape directory, with the limit settings applied. */
|
|
storageStatus(): MediaStorageStatus {
|
|
const prefs = this.deps.getMediaPrefs();
|
|
const byKind: Record<MediaKind, number> = { image: 0, gif: 0, video: 0, audio: 0 };
|
|
|
|
const scrapes = this.engine.list().map(meta => {
|
|
const dir = scrapeDir(this.rootDir, meta.id);
|
|
const bytes = dirSizeBytes(dir);
|
|
|
|
try {
|
|
const store = new ScrapeStore(dir);
|
|
try {
|
|
const sums = store.storedBytesByKind();
|
|
for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind];
|
|
} finally {
|
|
store.close();
|
|
}
|
|
} catch {
|
|
// an unreadable index counts as zero bytes
|
|
}
|
|
|
|
return { id: meta.id, channelId: meta.channelId, state: meta.state, createdAt: meta.createdAt, bytes };
|
|
});
|
|
|
|
const usedBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
|
|
|
|
return {
|
|
usedBytes,
|
|
limitBytes: prefs.limitBytes,
|
|
warnPercent: prefs.warnPercent,
|
|
overrunPercent: prefs.overrunPercent,
|
|
nearLimit: isNearLimit(usedBytes, prefs.limitBytes, prefs.warnPercent),
|
|
byKind,
|
|
scrapes
|
|
};
|
|
}
|
|
|
|
/** Mark the oldest completed scrapes when the usage is at or over the limit. */
|
|
refreshRetention(): MediaStorageStatus {
|
|
const status = this.storageStatus();
|
|
const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
|
|
for (const id of plan.mark) this.engine.setState(id, "pending-delete");
|
|
return this.storageStatus();
|
|
}
|
|
|
|
private applyRetention(): void {
|
|
const status = this.storageStatus();
|
|
const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
|
|
for (const id of plan.mark) this.engine.setState(id, "pending-delete");
|
|
for (const id of plan.delete) this.deleteScrape(id);
|
|
}
|
|
|
|
deleteScrape(id: string): boolean {
|
|
try {
|
|
rmSync(scrapeDir(this.rootDir, id), { recursive: true, force: true });
|
|
this.engine.forget(id);
|
|
this.deps.log?.(`deleted scrape ${id}`);
|
|
return true;
|
|
} catch (error) {
|
|
this.deps.log?.(`delete failed for ${id}: ${String(error)}`);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** Delete the given scrapes, or every scrape when no IDs are given. */
|
|
cleanMedia(ids?: string[]): number {
|
|
const targets = ids?.length ? ids : this.engine.list().map(meta => meta.id);
|
|
let deleted = 0;
|
|
for (const id of targets) {
|
|
if (this.deleteScrape(id)) deleted++;
|
|
}
|
|
return deleted;
|
|
}
|
|
|
|
async startScrape(input: StartScrapeInput): Promise<ScrapeMeta> {
|
|
if (input.downloadDir && !isAbsolute(input.downloadDir)) {
|
|
throw new Error("downloadDir must be an absolute path");
|
|
}
|
|
|
|
this.applyRetention();
|
|
|
|
const status = this.storageStatus();
|
|
if (status.nearLimit && !input.confirm) {
|
|
throw new MediaLimitError(
|
|
`media storage is at ${formatBytes(status.usedBytes)} of ${formatBytes(status.limitBytes)}`,
|
|
status.usedBytes,
|
|
status.limitBytes
|
|
);
|
|
}
|
|
|
|
try {
|
|
const mode = input.mode ?? "links";
|
|
const afterId = input.after ? snowflakeFromIso(input.after) : null;
|
|
const beforeId = input.before ? snowflakeFromIso(input.before) : null;
|
|
let backend: ScrapeBackend;
|
|
|
|
if (input.backend === "puppet") {
|
|
backend = this.puppetBackend();
|
|
} else {
|
|
const available = await this.ensureSlopbot();
|
|
const inGuild = !!input.guildId && (this.bot?.guilds.cache.has(input.guildId) ?? false);
|
|
|
|
if (input.backend === "slopbot" && (!available || !inGuild)) {
|
|
throw new Error("slopbot is not connected to that guild");
|
|
}
|
|
|
|
const fetcher = this.botFetcher;
|
|
backend =
|
|
available && inGuild && fetcher
|
|
? new SlopbotBackend(fetcher)
|
|
: this.puppetBackend();
|
|
}
|
|
|
|
// Best effort: one search request gives the total for the range.
|
|
let estimatedMessages: number | null = null;
|
|
if (backend.name === "slopbot" && input.guildId && this.bot) {
|
|
estimatedMessages = await countMessages(this.bot.rest as SlopbotRest, {
|
|
channelId: input.channelId,
|
|
guildId: input.guildId,
|
|
afterId,
|
|
beforeId
|
|
});
|
|
}
|
|
|
|
return this.engine.start({
|
|
channelId: input.channelId,
|
|
guildId: input.guildId ?? null,
|
|
mode,
|
|
backend,
|
|
downloadDir: input.downloadDir ?? null,
|
|
afterId,
|
|
beforeId,
|
|
fromDate: input.after ?? null,
|
|
toDate: input.before ?? null,
|
|
estimatedMessages
|
|
});
|
|
} catch (error) {
|
|
this.recordError("scrape", error instanceof Error ? error.message : String(error));
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
stopScrape(id: string): void {
|
|
this.engine.stop(id);
|
|
}
|
|
|
|
/**
|
|
* Title and channel from the source message's Discord embed. Discord keeps the embed data
|
|
* after a video is deleted, so this is the only title source for gone YouTube videos.
|
|
*/
|
|
private async embedMetaFor(
|
|
scrapeId: string,
|
|
media: { id: number; url: string; playUrl: string | null }
|
|
): Promise<{ title: string; author: string | null } | null> {
|
|
const meta = this.engine.get(scrapeId);
|
|
if (!meta) return null;
|
|
|
|
try {
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
|
|
let messageId: string | null = null;
|
|
try {
|
|
messageId = store.sourceMessageId(media.id);
|
|
} finally {
|
|
store.close();
|
|
}
|
|
if (!messageId) return null;
|
|
|
|
const message = await this.fetchSourceMessage(meta, messageId);
|
|
if (!message) return null;
|
|
|
|
const path = (url: string) => url.split("?")[0];
|
|
const embeds: any[] = message.embeds ?? [];
|
|
const embed = embeds.find(
|
|
candidate =>
|
|
(candidate?.url && path(candidate.url) === path(media.url)) ||
|
|
(media.playUrl && candidate?.videoUrl && path(candidate.videoUrl) === path(media.playUrl)) ||
|
|
(candidate?.video?.url && media.playUrl && path(candidate.video.url) === path(media.playUrl))
|
|
);
|
|
|
|
const title = embed?.title ?? null;
|
|
if (!title) return null;
|
|
const author = embed?.authorName ?? embed?.author?.name ?? null;
|
|
return { title, author };
|
|
} catch (error) {
|
|
this.deps.log?.(`embed meta failed for ${scrapeId}:${media.id}: ${String(error)}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** The source message through slopbot REST (guild scrapes) or the puppet bridge. */
|
|
private async fetchSourceMessage(meta: ScrapeMeta, messageId: string): Promise<any | null> {
|
|
if (this.bot && meta.guildId && meta.backend === "slopbot") {
|
|
const channel = await this.bot.channels.fetch(meta.channelId);
|
|
if (channel?.isTextBased()) return await channel.messages.fetch(messageId);
|
|
return null;
|
|
}
|
|
|
|
const core = this.deps.getCore();
|
|
if (!core || !this.deps.isCoreConnected()) return null;
|
|
|
|
const result = (await core.fetchMessages(meta.channelId, 1, {
|
|
before: (BigInt(messageId) + 1n).toString()
|
|
})) as { messages?: Array<Record<string, any>> };
|
|
|
|
return result?.messages?.find(candidate => candidate?.id === messageId) ?? result?.messages?.[0] ?? null;
|
|
}
|
|
|
|
/** Continue a paused scrape from its cursor with the same backend family. */
|
|
async resumeScrape(scrapeId: string): Promise<ScrapeMeta | null> {
|
|
const meta = this.engine.get(scrapeId);
|
|
if (!meta) return null;
|
|
|
|
let backend: ScrapeBackend;
|
|
if (meta.backend === "slopbot" && meta.guildId) {
|
|
const available = await this.ensureSlopbot();
|
|
const inGuild = this.bot?.guilds.cache.has(meta.guildId) ?? false;
|
|
const fetcher = this.botFetcher;
|
|
if (!available || !inGuild || !fetcher) {
|
|
throw new Error("slopbot is not connected to that guild");
|
|
}
|
|
backend = new SlopbotBackend(fetcher);
|
|
} else {
|
|
backend = this.puppetBackend();
|
|
}
|
|
|
|
this.deps.log?.(`resuming scrape ${scrapeId} from cursor ${meta.cursor ?? "start"}`);
|
|
return this.engine.resume(scrapeId, backend);
|
|
}
|
|
|
|
listScrapes(): ScrapeMeta[] {
|
|
return this.engine.list();
|
|
}
|
|
|
|
getScrape(id: string): ScrapeMeta | null {
|
|
return this.engine.get(id);
|
|
}
|
|
|
|
status(): MediaStatus {
|
|
const limit = this.engine.rateLimitStatus();
|
|
return {
|
|
slopbot: slopbotStatus(this.bot),
|
|
scrapes: this.engine.list(),
|
|
limitedUntil: limit.active
|
|
? new Date(limit.until ?? Date.now() + limit.remainingMs).toISOString()
|
|
: null,
|
|
providerFilters: [...PROVIDER_FILTERS]
|
|
};
|
|
}
|
|
|
|
/**
|
|
* A servable media source: the local file, or a remote URL refreshed from the source
|
|
* message (signed CDN links expire). Null when nothing can be resolved.
|
|
*/
|
|
async resolveMediaSource(
|
|
scrapeId: string,
|
|
itemId: number
|
|
): Promise<
|
|
| { kind: "file"; path: string; contentType: string }
|
|
| { kind: "remote"; url: string }
|
|
| null
|
|
> {
|
|
const file = this.getMediaFile(scrapeId, itemId);
|
|
if (file) return { kind: "file", ...file };
|
|
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item) return null;
|
|
|
|
return { kind: "remote", url: item.play_url ?? item.url };
|
|
}
|
|
|
|
/** Re-fetch the source message for a fresh signed URL. Null when not possible. */
|
|
async refreshMediaUrl(scrapeId: string, itemId: number): Promise<string | null> {
|
|
const meta = this.engine.get(scrapeId);
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!meta || !item) return null;
|
|
|
|
return this.refreshUrls(meta, item);
|
|
}
|
|
|
|
/**
|
|
* Re-fetch the source message and update expired URLs. Returns the fresh playable URL.
|
|
* Slopbot scrapes use the bot REST; DM and group-DM scrapes go through the core bridge.
|
|
*/
|
|
private async refreshUrls(meta: ScrapeMeta, item: MediaRow): Promise<string | null> {
|
|
try {
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
let messageId: string | null = null;
|
|
try {
|
|
messageId = store.sourceMessageId(item.id);
|
|
} finally {
|
|
store.close();
|
|
}
|
|
if (!messageId) return null;
|
|
|
|
const path = (url: string) => url.split("?")[0];
|
|
const saveAndReturn = (update: { url?: string; playUrl?: string | null }, fresh: string) => {
|
|
const writer = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
writer.updateUrls(item.id, update);
|
|
} finally {
|
|
writer.close();
|
|
}
|
|
return fresh;
|
|
};
|
|
|
|
if (this.bot && meta.guildId && meta.backend === "slopbot") {
|
|
const channel = await this.bot.channels.fetch(meta.channelId);
|
|
if (channel?.isTextBased()) {
|
|
const message = await channel.messages.fetch(messageId);
|
|
|
|
const attachment = [...message.attachments.values()].find(
|
|
candidate => path(candidate.url) === path(item.url)
|
|
);
|
|
if (attachment) return saveAndReturn({ url: attachment.url }, attachment.url);
|
|
|
|
const embed = message.embeds.find(
|
|
candidate =>
|
|
(item.play_url &&
|
|
candidate.video?.url &&
|
|
path(candidate.video.url) === path(item.play_url)) ||
|
|
(candidate.video?.url && path(candidate.url ?? "") === path(item.url))
|
|
);
|
|
if (embed?.video?.url) {
|
|
// The direct media link is always fresh; a bare embed.url can be stale.
|
|
return saveAndReturn(
|
|
{ playUrl: embed.video.url, url: embed.url ?? embed.video.url },
|
|
embed.video.url
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// The puppet path: the newest message below id+1 is exactly the target message.
|
|
const core = this.deps.getCore();
|
|
if (core && this.deps.isCoreConnected()) {
|
|
const result = (await core.fetchMessages(meta.channelId, 1, {
|
|
before: (BigInt(messageId) + 1n).toString()
|
|
})) as { messages?: Array<Record<string, any>> };
|
|
const message =
|
|
result?.messages?.find(candidate => candidate?.id === messageId) ??
|
|
result?.messages?.[0];
|
|
|
|
if (message) {
|
|
const attachment = (message.attachments ?? []).find(
|
|
(candidate: any) => candidate?.url && path(candidate.url) === path(item.url)
|
|
);
|
|
if (attachment?.url) return saveAndReturn({ url: attachment.url }, attachment.url);
|
|
|
|
const embed = (message.embeds ?? []).find(
|
|
(candidate: any) =>
|
|
(item.play_url &&
|
|
candidate?.videoUrl &&
|
|
path(candidate.videoUrl) === path(item.play_url)) ||
|
|
(candidate?.videoUrl && path(candidate.url ?? "") === path(item.url))
|
|
);
|
|
if (embed?.videoUrl) {
|
|
// Same as the bot path: keep the canonical url fresh too.
|
|
return saveAndReturn(
|
|
{ playUrl: embed.videoUrl, url: embed.url ?? embed.videoUrl },
|
|
embed.videoUrl
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
return null;
|
|
} catch (error) {
|
|
this.deps.log?.(`refresh failed for ${meta.id}:${item.id}: ${String(error)}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Absolute path and content type for a downloaded file, or null. */
|
|
getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
|
|
// A custom destination is the user's filesystem, outside the archive: not served.
|
|
const meta = this.engine.get(scrapeId);
|
|
if (meta?.downloadDir) return null;
|
|
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item?.file_path) return null;
|
|
|
|
const extension = item.file_path.split(".").pop()?.toLowerCase() ?? "";
|
|
return {
|
|
path: join(this.rootDir, scrapeId, item.file_path),
|
|
contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream"
|
|
};
|
|
}
|
|
|
|
/** Absolute path for a cached animated GIF, or null. */
|
|
getMediaAnim(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item?.anim_hash) return null;
|
|
|
|
return {
|
|
path: join(this.rootDir, scrapeId, "thumbs", `${item.anim_hash}.gif`),
|
|
contentType: "image/gif"
|
|
};
|
|
}
|
|
|
|
/** Absolute path for a cached preview, or null. */
|
|
getMediaThumb(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item?.preview_hash) return null;
|
|
|
|
const extension = item.preview_ext ?? "webp";
|
|
return {
|
|
path: join(this.rootDir, scrapeId, "thumbs", `${item.preview_hash}.${extension}`),
|
|
contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream"
|
|
};
|
|
}
|
|
|
|
/** Discord search total for a channel or date range. Slopbot (guild) channels only. */
|
|
async countMessages(input: {
|
|
channelId: string;
|
|
guildId?: string | null;
|
|
after?: string | null;
|
|
before?: string | null;
|
|
}): Promise<{ count: number | null; reason?: string }> {
|
|
if (!input.guildId) {
|
|
return { count: null, reason: "not available for DMs or group chats" };
|
|
}
|
|
|
|
const available = await this.ensureSlopbot();
|
|
if (!available || !this.bot) {
|
|
return { count: null, reason: "slopbot is not connected" };
|
|
}
|
|
|
|
const count = await countMessages(this.bot.rest as SlopbotRest, {
|
|
channelId: input.channelId,
|
|
guildId: input.guildId,
|
|
afterId: input.after ? snowflakeFromIso(input.after) : null,
|
|
beforeId: input.before ? snowflakeFromIso(input.before) : null
|
|
});
|
|
|
|
return count === null
|
|
? { count: null, reason: "search unavailable or rate limited" }
|
|
: { count };
|
|
}
|
|
|
|
private recordError(source: string, message: string): void {
|
|
this.errors.unshift({ at: new Date().toISOString(), source, message });
|
|
if (this.errors.length > 50) this.errors.length = 50;
|
|
}
|
|
|
|
/** Bot and scrape errors, newest first. */
|
|
listErrors(): MediaErrorEntry[] {
|
|
return [...this.errors];
|
|
}
|
|
|
|
clearErrors(): number {
|
|
const count = this.errors.length;
|
|
this.errors.length = 0;
|
|
return count;
|
|
}
|
|
|
|
/** Item counts for the status row, under the same filters as queryMedia. */
|
|
stats(options: {
|
|
kind?: MediaKind;
|
|
source?: string;
|
|
channelId?: string;
|
|
authorId?: string;
|
|
authorIds?: string[];
|
|
origin?: "dm" | "guild";
|
|
cachedOnly?: boolean;
|
|
exts?: string[];
|
|
excludeHosts?: string[];
|
|
excludeExts?: string[];
|
|
excludeProviders?: string[];
|
|
excludeSources?: string[];
|
|
includeProviders?: string[];
|
|
includeSources?: string[];
|
|
titleQuery?: string;
|
|
titleExact?: boolean;
|
|
sizeLessThan?: number;
|
|
sizeGreaterThan?: number;
|
|
lengthLessThan?: number;
|
|
lengthGreaterThan?: number;
|
|
sort?: MediaSort;
|
|
aspect?: string;
|
|
resolution?: string;
|
|
after?: string;
|
|
before?: string;
|
|
} = {}): Record<MediaKind, { total: number; downloaded: number; cached: number; notDownloaded: number }> {
|
|
// Same richest-per-URL dedupe as queryMedia.
|
|
const candidates = this.collectCandidates(options);
|
|
|
|
const totals: Record<
|
|
MediaKind,
|
|
{ total: number; downloaded: number; cached: number; notDownloaded: number }
|
|
> = {
|
|
image: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 },
|
|
gif: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 },
|
|
video: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 },
|
|
audio: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 }
|
|
};
|
|
|
|
for (const { row } of candidates) {
|
|
const bucket = totals[row.kind];
|
|
bucket.total++;
|
|
if (row.downloaded === 1) {
|
|
bucket.downloaded++;
|
|
} else {
|
|
bucket.notDownloaded++;
|
|
if (row.preview_hash !== null) bucket.cached++;
|
|
}
|
|
}
|
|
|
|
return totals;
|
|
}
|
|
|
|
/** Preview failures across scrapes with their reasons, newest scrape first. */
|
|
previewFailures(
|
|
limit = 50
|
|
): Array<{
|
|
scrapeId: string;
|
|
itemId: number;
|
|
kind: MediaKind;
|
|
provider: string | null;
|
|
url: string;
|
|
error: string;
|
|
}> {
|
|
const results: Array<{
|
|
scrapeId: string;
|
|
itemId: number;
|
|
kind: MediaKind;
|
|
provider: string | null;
|
|
url: string;
|
|
error: string;
|
|
}> = [];
|
|
|
|
for (const meta of this.engine.list()) {
|
|
if (results.length >= limit) break;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
for (const row of store.previewFailures(limit - results.length)) {
|
|
results.push({ scrapeId: meta.id, itemId: row.id, ...row });
|
|
}
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/** Distinct posters across every scrape, biggest first. */
|
|
listAuthors(): Array<{ id: string | null; name: string | null; count: number }> {
|
|
const totals = new Map<string, { id: string | null; name: string | null; count: number }>();
|
|
|
|
for (const meta of this.engine.list()) {
|
|
if (meta.state === "deleted") continue;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
for (const author of store.authors()) {
|
|
const key = author.id ?? author.name ?? "unknown";
|
|
const current = totals.get(key);
|
|
if (current) current.count += author.count;
|
|
else totals.set(key, { ...author });
|
|
}
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
return [...totals.values()].sort((a, b) => b.count - a.count);
|
|
}
|
|
|
|
/** One media row inside a scrape, with its scrape and channel for context. */
|
|
getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null {
|
|
const meta = this.engine.get(scrapeId);
|
|
if (!meta) return null;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
|
|
try {
|
|
const row = store.getMedia(itemId);
|
|
return row ? { ...row, scrapeId, channelId: meta.channelId } : null;
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* One entry per URL across every scrape. Scrapes are visited newest first, so a newer row
|
|
* wins by default; an older row replaces it only when it is richer (downloaded, or with a
|
|
* cached preview). Without this, a stopped or partial scrape hides usable previews.
|
|
*/
|
|
private collectCandidates(options: {
|
|
kind?: MediaKind;
|
|
source?: string;
|
|
scrapeId?: string;
|
|
channelId?: string;
|
|
authorId?: string;
|
|
authorIds?: string[];
|
|
origin?: "dm" | "guild";
|
|
cachedOnly?: boolean;
|
|
exts?: string[];
|
|
excludeHosts?: string[];
|
|
excludeExts?: string[];
|
|
excludeProviders?: string[];
|
|
excludeSources?: string[];
|
|
includeProviders?: string[];
|
|
includeSources?: string[];
|
|
titleQuery?: string;
|
|
titleExact?: boolean;
|
|
sizeLessThan?: number;
|
|
sizeGreaterThan?: number;
|
|
lengthLessThan?: number;
|
|
lengthGreaterThan?: number;
|
|
aspect?: string;
|
|
resolution?: string;
|
|
after?: string;
|
|
before?: string;
|
|
sort?: MediaSort;
|
|
}): Array<{ meta: ScrapeMeta; row: MediaRow }> {
|
|
const candidates = new Map<string, { meta: ScrapeMeta; row: MediaRow }>();
|
|
|
|
for (const meta of this.engine.list()) {
|
|
if (meta.state === "deleted") continue;
|
|
if (options.scrapeId && meta.id !== options.scrapeId) continue;
|
|
if (options.channelId && meta.channelId !== options.channelId) continue;
|
|
if (options.origin === "dm" && meta.guildId) continue;
|
|
if (options.origin === "guild" && !meta.guildId) continue;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
let storeOffset = 0;
|
|
|
|
for (;;) {
|
|
const rows = store.queryMedia({
|
|
kind: options.kind,
|
|
source: options.source,
|
|
authorId: options.authorId,
|
|
authorIds: options.authorIds,
|
|
cachedOnly: options.cachedOnly,
|
|
exts: options.exts,
|
|
excludeHosts: options.excludeHosts,
|
|
excludeExts: options.excludeExts,
|
|
excludeProviders: options.excludeProviders,
|
|
excludeSources: options.excludeSources,
|
|
includeProviders: options.includeProviders,
|
|
includeSources: options.includeSources,
|
|
titleQuery: options.titleQuery,
|
|
titleExact: options.titleExact,
|
|
sizeLessThan: options.sizeLessThan,
|
|
sizeGreaterThan: options.sizeGreaterThan,
|
|
lengthLessThan: options.lengthLessThan,
|
|
lengthGreaterThan: options.lengthGreaterThan,
|
|
aspect: options.aspect,
|
|
resolution: options.resolution,
|
|
after: options.after,
|
|
before: options.before,
|
|
limit: 500,
|
|
offset: storeOffset
|
|
});
|
|
if (!rows.length) break;
|
|
storeOffset += rows.length;
|
|
|
|
for (const row of rows) {
|
|
const current = candidates.get(row.url);
|
|
if (!current) {
|
|
candidates.set(row.url, { meta, row });
|
|
continue;
|
|
}
|
|
|
|
if (
|
|
betterCandidate(
|
|
{
|
|
downloaded: current.row.downloaded === 1,
|
|
hasPreview: current.row.preview_hash !== null
|
|
},
|
|
{ downloaded: row.downloaded === 1, hasPreview: row.preview_hash !== null }
|
|
)
|
|
) {
|
|
candidates.set(row.url, { meta, row });
|
|
}
|
|
}
|
|
|
|
if (rows.length < 500) break;
|
|
}
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
return sortCandidates([...candidates.values()], options.sort ?? "newest");
|
|
}
|
|
|
|
/** Read media rows across every scrape, richest per URL, newest scrape first. */
|
|
queryMedia(
|
|
options: {
|
|
kind?: MediaKind;
|
|
source?: string;
|
|
scrapeId?: string;
|
|
channelId?: string;
|
|
authorId?: string;
|
|
authorIds?: string[];
|
|
origin?: "dm" | "guild";
|
|
cachedOnly?: boolean;
|
|
exts?: string[];
|
|
excludeHosts?: string[];
|
|
excludeExts?: string[];
|
|
excludeProviders?: string[];
|
|
excludeSources?: string[];
|
|
includeProviders?: string[];
|
|
includeSources?: string[];
|
|
titleQuery?: string;
|
|
titleExact?: boolean;
|
|
sizeLessThan?: number;
|
|
sizeGreaterThan?: number;
|
|
lengthLessThan?: number;
|
|
lengthGreaterThan?: number;
|
|
sort?: MediaSort;
|
|
aspect?: string;
|
|
resolution?: string;
|
|
after?: string;
|
|
before?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
} = {}
|
|
): MediaItemResult[] {
|
|
const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
|
|
const offset = Math.max(0, options.offset ?? 0);
|
|
|
|
return this.collectCandidates(options)
|
|
.slice(offset, offset + limit)
|
|
.map(({ meta, row }) => ({
|
|
scrapeId: meta.id,
|
|
itemId: row.id,
|
|
channelId: meta.channelId,
|
|
guildId: meta.guildId,
|
|
messageId: row.message_id,
|
|
messageUrl: messageUrlFor(meta.guildId, meta.channelId, row.message_id),
|
|
kind: row.kind,
|
|
source: row.source,
|
|
authorId: row.author_id,
|
|
authorName: row.author_name,
|
|
postedAt: row.posted_at,
|
|
width: row.width,
|
|
height: row.height,
|
|
aspectRatio: row.aspect_ratio,
|
|
contentType: row.content_type,
|
|
durationSecs: row.duration_secs,
|
|
url: row.url,
|
|
playUrl: row.play_url,
|
|
remotePreviewUrl: row.preview_url,
|
|
previewError: row.preview_error,
|
|
titleError: row.title_error,
|
|
animUrl: row.anim_hash ? `/api/media/${meta.id}/${row.id}/anim` : null,
|
|
localUrl:
|
|
row.file_path && !meta.downloadDir
|
|
? `/api/media/${meta.id}/${row.id}/file`
|
|
: null,
|
|
thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
|
|
provider: row.provider,
|
|
downloaded: row.downloaded === 1,
|
|
title: row.title,
|
|
titleAuthor: row.title_author,
|
|
createdAt: row.created_at
|
|
}));
|
|
}
|
|
}
|