879 lines
32 KiB
TypeScript
879 lines
32 KiB
TypeScript
/*
|
|
* Media archive service: the daemon side of media-scrape/. See docs/media.md.
|
|
*
|
|
* Owns the scrape engine, the slopbot client, and the read path the Web UI and
|
|
* (later) MCP tools call.
|
|
*/
|
|
|
|
import { rmSync } from "node:fs";
|
|
import { isAbsolute, join } from "node:path";
|
|
import {
|
|
betterCandidate,
|
|
dirSizeBytes,
|
|
isNearLimit,
|
|
planRetention,
|
|
ScrapeEngine,
|
|
ScrapeStore,
|
|
scrapeDir,
|
|
snowflakeFromIso,
|
|
resolveDbRoot,
|
|
type MediaKind,
|
|
type MediaRow,
|
|
type ScrapeBackend,
|
|
type ScrapeMeta
|
|
} from "../../media-scrape/src/index.js";
|
|
import { PuppetBackend } from "../../media-scrape/src/puppet-backend.js";
|
|
import {
|
|
countMessages,
|
|
loadSlopbotConfig,
|
|
slopbotFetcher,
|
|
slopbotStatus,
|
|
SlopbotBackend,
|
|
startSlopbot,
|
|
type SlopbotFetcher,
|
|
type SlopbotRest,
|
|
type SlopbotStatus
|
|
} from "../../media-scrape/slopbot/index.js";
|
|
import type { CoreClient } from "./core-client.js";
|
|
import type { MediaPreferences } from "./preferences.js";
|
|
|
|
type BotClient = Awaited<ReturnType<typeof startSlopbot>>;
|
|
|
|
export interface MediaServiceDeps {
|
|
getCore: () => CoreClient | undefined;
|
|
isCoreConnected: () => boolean;
|
|
getMediaPrefs: () => MediaPreferences;
|
|
log?: (line: string) => void;
|
|
}
|
|
|
|
export interface StartScrapeInput {
|
|
channelId: string;
|
|
guildId?: string | null;
|
|
mode?: "links" | "download";
|
|
backend?: "auto" | "slopbot" | "puppet";
|
|
/** Set by a re-run of the command after the near-limit warning. */
|
|
confirm?: boolean;
|
|
/** Absolute destination for downloaded files. Null keeps them under the scrape dir. */
|
|
downloadDir?: string | null;
|
|
/** From-date as ISO; the walk stops below it. */
|
|
after?: string | null;
|
|
/** To-date as ISO; the walk starts just below it. */
|
|
before?: string | null;
|
|
}
|
|
|
|
export interface MediaErrorEntry {
|
|
at: string;
|
|
source: string;
|
|
message: string;
|
|
}
|
|
|
|
export interface ScrapeBytes {
|
|
id: string;
|
|
channelId: string;
|
|
state: string;
|
|
createdAt: string;
|
|
bytes: number;
|
|
}
|
|
|
|
export interface MediaStorageStatus {
|
|
usedBytes: number;
|
|
limitBytes: number;
|
|
warnPercent: number;
|
|
overrunPercent: number;
|
|
nearLimit: boolean;
|
|
byKind: Record<MediaKind, number>;
|
|
scrapes: ScrapeBytes[];
|
|
}
|
|
|
|
/** Thrown when a scrape starts near the storage limit without confirmation. */
|
|
export class MediaLimitError extends Error {
|
|
readonly code = "MEDIA_LIMIT";
|
|
|
|
constructor(
|
|
message: string,
|
|
readonly usedBytes: number,
|
|
readonly limitBytes: number
|
|
) {
|
|
super(message);
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes: number): string {
|
|
const gib = bytes / 1024 ** 3;
|
|
return gib >= 1 ? `${gib.toFixed(2)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`;
|
|
}
|
|
|
|
export interface MediaItemResult {
|
|
scrapeId: string;
|
|
itemId: number;
|
|
channelId: string;
|
|
kind: MediaKind;
|
|
source: string | null;
|
|
authorId: string | null;
|
|
authorName: string | null;
|
|
postedAt: string | null;
|
|
width: number | null;
|
|
height: number | null;
|
|
aspectRatio: number | null;
|
|
contentType: string | null;
|
|
durationSecs: number | null;
|
|
url: string;
|
|
playUrl: string | null;
|
|
/** The remote thumbnail URL from Discord's embed data (may be an animated GIF). */
|
|
remotePreviewUrl: string | null;
|
|
/** Why no preview exists, when a fetch failed (for example "http 404"). */
|
|
previewError: string | null;
|
|
/** Local animated GIF for hover playback, when the original was cached. */
|
|
animUrl: string | null;
|
|
localUrl: string | null;
|
|
thumbUrl: string | null;
|
|
provider: string | null;
|
|
downloaded: boolean;
|
|
createdAt: string;
|
|
}
|
|
|
|
const FILE_CONTENT_TYPES: Record<string, string> = {
|
|
png: "image/png",
|
|
jpg: "image/jpeg",
|
|
jpeg: "image/jpeg",
|
|
gif: "image/gif",
|
|
webp: "image/webp",
|
|
avif: "image/avif",
|
|
mp4: "video/mp4",
|
|
webm: "video/webm",
|
|
mov: "video/quicktime",
|
|
mkv: "video/x-matroska",
|
|
mp3: "audio/mpeg",
|
|
ogg: "audio/ogg",
|
|
wav: "audio/wav",
|
|
m4a: "audio/mp4",
|
|
flac: "audio/flac"
|
|
};
|
|
|
|
export interface MediaStatus {
|
|
slopbot: SlopbotStatus;
|
|
scrapes: ScrapeMeta[];
|
|
}
|
|
|
|
export class MediaService {
|
|
private readonly rootDir = resolveDbRoot();
|
|
private readonly engine: ScrapeEngine;
|
|
|
|
private bot: BotClient | null = null;
|
|
private botFetcher: SlopbotFetcher | null = null;
|
|
private botStarting: Promise<void> | null = null;
|
|
|
|
/** The newest bot and scrape errors, capped. Surfaced in the Bot tab. */
|
|
private readonly errors: MediaErrorEntry[] = [];
|
|
|
|
constructor(private readonly deps: MediaServiceDeps) {
|
|
const number = (value: string | undefined, fallback: number) => {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
|
};
|
|
|
|
this.engine = new ScrapeEngine({
|
|
rootDir: this.rootDir,
|
|
log: line => this.deps.log?.(line),
|
|
onError: (scrapeId, message) => this.recordError("scrape", `${scrapeId}: ${message}`),
|
|
walkCyclePages: number(process.env.VESKSLOP_MEDIA_WALK_CYCLE_PAGES, 5),
|
|
rateLimitPauseMs: number(process.env.VESKSLOP_MEDIA_RATELIMIT_PAUSE_MS, 60_000),
|
|
download: {
|
|
maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
|
|
previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),
|
|
previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200),
|
|
previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600),
|
|
previewBatchSize: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH, 25),
|
|
previewBatchPauseMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_BATCH_PAUSE_MS, 60_000),
|
|
minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
|
|
maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
|
|
}
|
|
});
|
|
|
|
// Preview passes only run when a walk ends; fill in scrapes that paused early.
|
|
void this.backfillPreviews();
|
|
}
|
|
|
|
private async backfillPreviews(): Promise<void> {
|
|
for (const meta of this.engine.list()) {
|
|
if (meta.state === "running" || meta.state === "queued") continue;
|
|
|
|
try {
|
|
const added = await this.engine.backfillPreviews(meta.id);
|
|
if (added) this.deps.log?.(`backfilled ${added} previews for ${meta.id}`);
|
|
} catch (error) {
|
|
this.deps.log?.(`preview backfill failed for ${meta.id}: ${String(error)}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Log slopbot in on first use. False when it is disabled or the login failed. */
|
|
private async ensureSlopbot(): Promise<boolean> {
|
|
if (this.bot) return true;
|
|
|
|
const config = loadSlopbotConfig();
|
|
if (!config.enabled) return false;
|
|
|
|
if (!this.botStarting) {
|
|
this.botStarting = startSlopbot(config)
|
|
.then(client => {
|
|
this.bot = client;
|
|
this.botFetcher = slopbotFetcher(client);
|
|
this.deps.log?.("slopbot connected");
|
|
})
|
|
.catch(error => {
|
|
const message = `slopbot login failed: ${String(error)}`;
|
|
this.deps.log?.(message);
|
|
this.recordError("slopbot", message);
|
|
});
|
|
}
|
|
|
|
await this.botStarting;
|
|
return this.bot !== null;
|
|
}
|
|
|
|
private puppetBackend(): ScrapeBackend {
|
|
return new PuppetBackend(async (channelId, limit, options) => {
|
|
const core = this.deps.getCore();
|
|
if (!core || !this.deps.isCoreConnected()) {
|
|
throw new Error("the core bridge is not connected");
|
|
}
|
|
return (await core.fetchMessages(channelId, limit, options)) as { messages?: unknown[] };
|
|
});
|
|
}
|
|
|
|
/** Combined size of every scrape directory, with the limit settings applied. */
|
|
storageStatus(): MediaStorageStatus {
|
|
const prefs = this.deps.getMediaPrefs();
|
|
const byKind: Record<MediaKind, number> = { image: 0, gif: 0, video: 0, audio: 0 };
|
|
|
|
const scrapes = this.engine.list().map(meta => {
|
|
const dir = scrapeDir(this.rootDir, meta.id);
|
|
const bytes = dirSizeBytes(dir);
|
|
|
|
try {
|
|
const store = new ScrapeStore(dir);
|
|
try {
|
|
const sums = store.storedBytesByKind();
|
|
for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind];
|
|
} finally {
|
|
store.close();
|
|
}
|
|
} catch {
|
|
// an unreadable index counts as zero bytes
|
|
}
|
|
|
|
return { id: meta.id, channelId: meta.channelId, state: meta.state, createdAt: meta.createdAt, bytes };
|
|
});
|
|
|
|
const usedBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
|
|
|
|
return {
|
|
usedBytes,
|
|
limitBytes: prefs.limitBytes,
|
|
warnPercent: prefs.warnPercent,
|
|
overrunPercent: prefs.overrunPercent,
|
|
nearLimit: isNearLimit(usedBytes, prefs.limitBytes, prefs.warnPercent),
|
|
byKind,
|
|
scrapes
|
|
};
|
|
}
|
|
|
|
/** Mark the oldest completed scrapes when the usage is at or over the limit. */
|
|
refreshRetention(): MediaStorageStatus {
|
|
const status = this.storageStatus();
|
|
const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
|
|
for (const id of plan.mark) this.engine.setState(id, "pending-delete");
|
|
return this.storageStatus();
|
|
}
|
|
|
|
private applyRetention(): void {
|
|
const status = this.storageStatus();
|
|
const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
|
|
for (const id of plan.mark) this.engine.setState(id, "pending-delete");
|
|
for (const id of plan.delete) this.deleteScrape(id);
|
|
}
|
|
|
|
deleteScrape(id: string): boolean {
|
|
try {
|
|
rmSync(scrapeDir(this.rootDir, id), { recursive: true, force: true });
|
|
this.engine.forget(id);
|
|
this.deps.log?.(`deleted scrape ${id}`);
|
|
return true;
|
|
} catch (error) {
|
|
this.deps.log?.(`delete failed for ${id}: ${String(error)}`);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** Delete the given scrapes, or every scrape when no IDs are given. */
|
|
cleanMedia(ids?: string[]): number {
|
|
const targets = ids?.length ? ids : this.engine.list().map(meta => meta.id);
|
|
let deleted = 0;
|
|
for (const id of targets) {
|
|
if (this.deleteScrape(id)) deleted++;
|
|
}
|
|
return deleted;
|
|
}
|
|
|
|
async startScrape(input: StartScrapeInput): Promise<ScrapeMeta> {
|
|
if (input.downloadDir && !isAbsolute(input.downloadDir)) {
|
|
throw new Error("downloadDir must be an absolute path");
|
|
}
|
|
|
|
this.applyRetention();
|
|
|
|
const status = this.storageStatus();
|
|
if (status.nearLimit && !input.confirm) {
|
|
throw new MediaLimitError(
|
|
`media storage is at ${formatBytes(status.usedBytes)} of ${formatBytes(status.limitBytes)}`,
|
|
status.usedBytes,
|
|
status.limitBytes
|
|
);
|
|
}
|
|
|
|
try {
|
|
const mode = input.mode ?? "links";
|
|
const afterId = input.after ? snowflakeFromIso(input.after) : null;
|
|
const beforeId = input.before ? snowflakeFromIso(input.before) : null;
|
|
let backend: ScrapeBackend;
|
|
|
|
if (input.backend === "puppet") {
|
|
backend = this.puppetBackend();
|
|
} else {
|
|
const available = await this.ensureSlopbot();
|
|
const inGuild = !!input.guildId && (this.bot?.guilds.cache.has(input.guildId) ?? false);
|
|
|
|
if (input.backend === "slopbot" && (!available || !inGuild)) {
|
|
throw new Error("slopbot is not connected to that guild");
|
|
}
|
|
|
|
const fetcher = this.botFetcher;
|
|
backend =
|
|
available && inGuild && fetcher
|
|
? new SlopbotBackend(fetcher)
|
|
: this.puppetBackend();
|
|
}
|
|
|
|
// Best effort: one search request gives the total for the range.
|
|
let estimatedMessages: number | null = null;
|
|
if (backend.name === "slopbot" && input.guildId && this.bot) {
|
|
estimatedMessages = await countMessages(this.bot.rest as SlopbotRest, {
|
|
channelId: input.channelId,
|
|
guildId: input.guildId,
|
|
afterId,
|
|
beforeId
|
|
});
|
|
}
|
|
|
|
return this.engine.start({
|
|
channelId: input.channelId,
|
|
guildId: input.guildId ?? null,
|
|
mode,
|
|
backend,
|
|
downloadDir: input.downloadDir ?? null,
|
|
afterId,
|
|
beforeId,
|
|
fromDate: input.after ?? null,
|
|
toDate: input.before ?? null,
|
|
estimatedMessages
|
|
});
|
|
} catch (error) {
|
|
this.recordError("scrape", error instanceof Error ? error.message : String(error));
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
stopScrape(id: string): void {
|
|
this.engine.stop(id);
|
|
}
|
|
|
|
/** Continue a paused scrape from its cursor with the same backend family. */
|
|
async resumeScrape(scrapeId: string): Promise<ScrapeMeta | null> {
|
|
const meta = this.engine.get(scrapeId);
|
|
if (!meta) return null;
|
|
|
|
let backend: ScrapeBackend;
|
|
if (meta.backend === "slopbot" && meta.guildId) {
|
|
const available = await this.ensureSlopbot();
|
|
const inGuild = this.bot?.guilds.cache.has(meta.guildId) ?? false;
|
|
const fetcher = this.botFetcher;
|
|
if (!available || !inGuild || !fetcher) {
|
|
throw new Error("slopbot is not connected to that guild");
|
|
}
|
|
backend = new SlopbotBackend(fetcher);
|
|
} else {
|
|
backend = this.puppetBackend();
|
|
}
|
|
|
|
this.deps.log?.(`resuming scrape ${scrapeId} from cursor ${meta.cursor ?? "start"}`);
|
|
return this.engine.resume(scrapeId, backend);
|
|
}
|
|
|
|
listScrapes(): ScrapeMeta[] {
|
|
return this.engine.list();
|
|
}
|
|
|
|
getScrape(id: string): ScrapeMeta | null {
|
|
return this.engine.get(id);
|
|
}
|
|
|
|
status(): MediaStatus {
|
|
return {
|
|
slopbot: slopbotStatus(this.bot),
|
|
scrapes: this.engine.list()
|
|
};
|
|
}
|
|
|
|
/**
|
|
* A servable media source: the local file, or a remote URL refreshed from the source
|
|
* message (signed CDN links expire). Null when nothing can be resolved.
|
|
*/
|
|
async resolveMediaSource(
|
|
scrapeId: string,
|
|
itemId: number
|
|
): Promise<
|
|
| { kind: "file"; path: string; contentType: string }
|
|
| { kind: "remote"; url: string }
|
|
| null
|
|
> {
|
|
const file = this.getMediaFile(scrapeId, itemId);
|
|
if (file) return { kind: "file", ...file };
|
|
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item) return null;
|
|
|
|
return { kind: "remote", url: item.play_url ?? item.url };
|
|
}
|
|
|
|
/** Re-fetch the source message for a fresh signed URL. Null when not possible. */
|
|
async refreshMediaUrl(scrapeId: string, itemId: number): Promise<string | null> {
|
|
const meta = this.engine.get(scrapeId);
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!meta || !item) return null;
|
|
|
|
return this.refreshUrls(meta, item);
|
|
}
|
|
|
|
/**
|
|
* Re-fetch the source message and update expired URLs. Returns the fresh playable URL.
|
|
* Slopbot scrapes use the bot REST; DM and group-DM scrapes go through the core bridge.
|
|
*/
|
|
private async refreshUrls(meta: ScrapeMeta, item: MediaRow): Promise<string | null> {
|
|
try {
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
let messageId: string | null = null;
|
|
try {
|
|
messageId = store.sourceMessageId(item.id);
|
|
} finally {
|
|
store.close();
|
|
}
|
|
if (!messageId) return null;
|
|
|
|
const path = (url: string) => url.split("?")[0];
|
|
const saveAndReturn = (update: { url?: string; playUrl?: string | null }, fresh: string) => {
|
|
const writer = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
writer.updateUrls(item.id, update);
|
|
} finally {
|
|
writer.close();
|
|
}
|
|
return fresh;
|
|
};
|
|
|
|
if (this.bot && meta.guildId && meta.backend === "slopbot") {
|
|
const channel = await this.bot.channels.fetch(meta.channelId);
|
|
if (channel?.isTextBased()) {
|
|
const message = await channel.messages.fetch(messageId);
|
|
|
|
const attachment = [...message.attachments.values()].find(
|
|
candidate => path(candidate.url) === path(item.url)
|
|
);
|
|
if (attachment) return saveAndReturn({ url: attachment.url }, attachment.url);
|
|
|
|
const embed = message.embeds.find(
|
|
candidate =>
|
|
(item.play_url &&
|
|
candidate.video?.url &&
|
|
path(candidate.video.url) === path(item.play_url)) ||
|
|
(candidate.video?.url && path(candidate.url ?? "") === path(item.url))
|
|
);
|
|
if (embed?.video?.url) {
|
|
return saveAndReturn(
|
|
{ playUrl: embed.video.url, url: embed.url ?? item.url },
|
|
embed.video.url
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// The puppet path: the newest message below id+1 is exactly the target message.
|
|
const core = this.deps.getCore();
|
|
if (core && this.deps.isCoreConnected()) {
|
|
const result = (await core.fetchMessages(meta.channelId, 1, {
|
|
before: (BigInt(messageId) + 1n).toString()
|
|
})) as { messages?: Array<Record<string, any>> };
|
|
const message =
|
|
result?.messages?.find(candidate => candidate?.id === messageId) ??
|
|
result?.messages?.[0];
|
|
|
|
if (message) {
|
|
const attachment = (message.attachments ?? []).find(
|
|
(candidate: any) => candidate?.url && path(candidate.url) === path(item.url)
|
|
);
|
|
if (attachment?.url) return saveAndReturn({ url: attachment.url }, attachment.url);
|
|
|
|
const embed = (message.embeds ?? []).find(
|
|
(candidate: any) =>
|
|
(item.play_url &&
|
|
candidate?.videoUrl &&
|
|
path(candidate.videoUrl) === path(item.play_url)) ||
|
|
(candidate?.videoUrl && path(candidate.url ?? "") === path(item.url))
|
|
);
|
|
if (embed?.videoUrl) {
|
|
return saveAndReturn(
|
|
{ playUrl: embed.videoUrl, url: embed.url ?? item.url },
|
|
embed.videoUrl
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
return null;
|
|
} catch (error) {
|
|
this.deps.log?.(`refresh failed for ${meta.id}:${item.id}: ${String(error)}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Absolute path and content type for a downloaded file, or null. */
|
|
getMediaFile(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
|
|
// A custom destination is the user's filesystem, outside the archive: not served.
|
|
const meta = this.engine.get(scrapeId);
|
|
if (meta?.downloadDir) return null;
|
|
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item?.file_path) return null;
|
|
|
|
const extension = item.file_path.split(".").pop()?.toLowerCase() ?? "";
|
|
return {
|
|
path: join(this.rootDir, scrapeId, item.file_path),
|
|
contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream"
|
|
};
|
|
}
|
|
|
|
/** Absolute path for a cached animated GIF, or null. */
|
|
getMediaAnim(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item?.anim_hash) return null;
|
|
|
|
return {
|
|
path: join(this.rootDir, scrapeId, "thumbs", `${item.anim_hash}.gif`),
|
|
contentType: "image/gif"
|
|
};
|
|
}
|
|
|
|
/** Absolute path for a cached preview, or null. */
|
|
getMediaThumb(scrapeId: string, itemId: number): { path: string; contentType: string } | null {
|
|
const item = this.getMediaItem(scrapeId, itemId);
|
|
if (!item?.preview_hash) return null;
|
|
|
|
const extension = item.preview_ext ?? "webp";
|
|
return {
|
|
path: join(this.rootDir, scrapeId, "thumbs", `${item.preview_hash}.${extension}`),
|
|
contentType: FILE_CONTENT_TYPES[extension] ?? "application/octet-stream"
|
|
};
|
|
}
|
|
|
|
/** Discord search total for a channel or date range. Slopbot (guild) channels only. */
|
|
async countMessages(input: {
|
|
channelId: string;
|
|
guildId?: string | null;
|
|
after?: string | null;
|
|
before?: string | null;
|
|
}): Promise<{ count: number | null; reason?: string }> {
|
|
if (!input.guildId) {
|
|
return { count: null, reason: "not available for DMs or group chats" };
|
|
}
|
|
|
|
const available = await this.ensureSlopbot();
|
|
if (!available || !this.bot) {
|
|
return { count: null, reason: "slopbot is not connected" };
|
|
}
|
|
|
|
const count = await countMessages(this.bot.rest as SlopbotRest, {
|
|
channelId: input.channelId,
|
|
guildId: input.guildId,
|
|
afterId: input.after ? snowflakeFromIso(input.after) : null,
|
|
beforeId: input.before ? snowflakeFromIso(input.before) : null
|
|
});
|
|
|
|
return count === null
|
|
? { count: null, reason: "search unavailable or rate limited" }
|
|
: { count };
|
|
}
|
|
|
|
private recordError(source: string, message: string): void {
|
|
this.errors.unshift({ at: new Date().toISOString(), source, message });
|
|
if (this.errors.length > 50) this.errors.length = 50;
|
|
}
|
|
|
|
/** Bot and scrape errors, newest first. */
|
|
listErrors(): MediaErrorEntry[] {
|
|
return [...this.errors];
|
|
}
|
|
|
|
clearErrors(): number {
|
|
const count = this.errors.length;
|
|
this.errors.length = 0;
|
|
return count;
|
|
}
|
|
|
|
/** Item counts for the status row, under the same filters as queryMedia. */
|
|
stats(options: {
|
|
kind?: MediaKind;
|
|
source?: string;
|
|
channelId?: string;
|
|
authorId?: string;
|
|
authorIds?: string[];
|
|
origin?: "dm" | "guild";
|
|
cachedOnly?: boolean;
|
|
aspect?: string;
|
|
resolution?: string;
|
|
after?: string;
|
|
before?: string;
|
|
} = {}): Record<MediaKind, { total: number; downloaded: number; cached: number; notDownloaded: number }> {
|
|
// Same richest-per-URL dedupe as queryMedia.
|
|
const candidates = this.collectCandidates(options);
|
|
|
|
const totals: Record<
|
|
MediaKind,
|
|
{ total: number; downloaded: number; cached: number; notDownloaded: number }
|
|
> = {
|
|
image: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 },
|
|
gif: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 },
|
|
video: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 },
|
|
audio: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 }
|
|
};
|
|
|
|
for (const { row } of candidates) {
|
|
const bucket = totals[row.kind];
|
|
bucket.total++;
|
|
if (row.downloaded === 1) {
|
|
bucket.downloaded++;
|
|
} else {
|
|
bucket.notDownloaded++;
|
|
if (row.preview_hash !== null) bucket.cached++;
|
|
}
|
|
}
|
|
|
|
return totals;
|
|
}
|
|
|
|
/** Preview failures across scrapes with their reasons, newest scrape first. */
|
|
previewFailures(
|
|
limit = 50
|
|
): Array<{
|
|
scrapeId: string;
|
|
itemId: number;
|
|
kind: MediaKind;
|
|
provider: string | null;
|
|
url: string;
|
|
error: string;
|
|
}> {
|
|
const results: Array<{
|
|
scrapeId: string;
|
|
itemId: number;
|
|
kind: MediaKind;
|
|
provider: string | null;
|
|
url: string;
|
|
error: string;
|
|
}> = [];
|
|
|
|
for (const meta of this.engine.list()) {
|
|
if (results.length >= limit) break;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
for (const row of store.previewFailures(limit - results.length)) {
|
|
results.push({ scrapeId: meta.id, itemId: row.id, ...row });
|
|
}
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/** Distinct posters across every scrape, biggest first. */
|
|
listAuthors(): Array<{ id: string | null; name: string | null; count: number }> {
|
|
const totals = new Map<string, { id: string | null; name: string | null; count: number }>();
|
|
|
|
for (const meta of this.engine.list()) {
|
|
if (meta.state === "deleted") continue;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
for (const author of store.authors()) {
|
|
const key = author.id ?? author.name ?? "unknown";
|
|
const current = totals.get(key);
|
|
if (current) current.count += author.count;
|
|
else totals.set(key, { ...author });
|
|
}
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
return [...totals.values()].sort((a, b) => b.count - a.count);
|
|
}
|
|
|
|
/** One media row inside a scrape, with its scrape and channel for context. */
|
|
getMediaItem(scrapeId: string, itemId: number): (MediaRow & { scrapeId: string; channelId: string }) | null {
|
|
const meta = this.engine.get(scrapeId);
|
|
if (!meta) return null;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, scrapeId));
|
|
try {
|
|
const row = store.getMedia(itemId);
|
|
return row ? { ...row, scrapeId, channelId: meta.channelId } : null;
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* One entry per URL across every scrape. Scrapes are visited newest first, so a newer row
|
|
* wins by default; an older row replaces it only when it is richer (downloaded, or with a
|
|
* cached preview). Without this, a stopped or partial scrape hides usable previews.
|
|
*/
|
|
private collectCandidates(options: {
|
|
kind?: MediaKind;
|
|
source?: string;
|
|
scrapeId?: string;
|
|
channelId?: string;
|
|
authorId?: string;
|
|
authorIds?: string[];
|
|
origin?: "dm" | "guild";
|
|
cachedOnly?: boolean;
|
|
aspect?: string;
|
|
resolution?: string;
|
|
after?: string;
|
|
before?: string;
|
|
}): Array<{ meta: ScrapeMeta; row: MediaRow }> {
|
|
const candidates = new Map<string, { meta: ScrapeMeta; row: MediaRow }>();
|
|
|
|
for (const meta of this.engine.list()) {
|
|
if (meta.state === "deleted") continue;
|
|
if (options.scrapeId && meta.id !== options.scrapeId) continue;
|
|
if (options.channelId && meta.channelId !== options.channelId) continue;
|
|
if (options.origin === "dm" && meta.guildId) continue;
|
|
if (options.origin === "guild" && !meta.guildId) continue;
|
|
|
|
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
|
try {
|
|
let storeOffset = 0;
|
|
|
|
for (;;) {
|
|
const rows = store.queryMedia({
|
|
kind: options.kind,
|
|
source: options.source,
|
|
authorId: options.authorId,
|
|
authorIds: options.authorIds,
|
|
cachedOnly: options.cachedOnly,
|
|
aspect: options.aspect,
|
|
resolution: options.resolution,
|
|
after: options.after,
|
|
before: options.before,
|
|
limit: 500,
|
|
offset: storeOffset
|
|
});
|
|
if (!rows.length) break;
|
|
storeOffset += rows.length;
|
|
|
|
for (const row of rows) {
|
|
const current = candidates.get(row.url);
|
|
if (!current) {
|
|
candidates.set(row.url, { meta, row });
|
|
continue;
|
|
}
|
|
|
|
if (
|
|
betterCandidate(
|
|
{
|
|
downloaded: current.row.downloaded === 1,
|
|
hasPreview: current.row.preview_hash !== null
|
|
},
|
|
{ downloaded: row.downloaded === 1, hasPreview: row.preview_hash !== null }
|
|
)
|
|
) {
|
|
candidates.set(row.url, { meta, row });
|
|
}
|
|
}
|
|
|
|
if (rows.length < 500) break;
|
|
}
|
|
} finally {
|
|
store.close();
|
|
}
|
|
}
|
|
|
|
return [...candidates.values()];
|
|
}
|
|
|
|
/** Read media rows across every scrape, richest per URL, newest scrape first. */
|
|
queryMedia(
|
|
options: {
|
|
kind?: MediaKind;
|
|
source?: string;
|
|
scrapeId?: string;
|
|
channelId?: string;
|
|
authorId?: string;
|
|
authorIds?: string[];
|
|
origin?: "dm" | "guild";
|
|
cachedOnly?: boolean;
|
|
aspect?: string;
|
|
resolution?: string;
|
|
after?: string;
|
|
before?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
} = {}
|
|
): MediaItemResult[] {
|
|
const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
|
|
const offset = Math.max(0, options.offset ?? 0);
|
|
|
|
return this.collectCandidates(options)
|
|
.slice(offset, offset + limit)
|
|
.map(({ meta, row }) => ({
|
|
scrapeId: meta.id,
|
|
itemId: row.id,
|
|
channelId: meta.channelId,
|
|
kind: row.kind,
|
|
source: row.source,
|
|
authorId: row.author_id,
|
|
authorName: row.author_name,
|
|
postedAt: row.posted_at,
|
|
width: row.width,
|
|
height: row.height,
|
|
aspectRatio: row.aspect_ratio,
|
|
contentType: row.content_type,
|
|
durationSecs: row.duration_secs,
|
|
url: row.url,
|
|
playUrl: row.play_url,
|
|
remotePreviewUrl: row.preview_url,
|
|
previewError: row.preview_error,
|
|
animUrl: row.anim_hash ? `/api/media/${meta.id}/${row.id}/anim` : null,
|
|
localUrl:
|
|
row.file_path && !meta.downloadDir
|
|
? `/api/media/${meta.id}/${row.id}/file`
|
|
: null,
|
|
thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
|
|
provider: row.provider,
|
|
downloaded: row.downloaded === 1,
|
|
createdAt: row.created_at
|
|
}));
|
|
}
|
|
}
|