zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add message links, suspect retries, and Discord preview recovery

Browse Source
This commit is contained in:
Fisto
2026-09-18 11:47:08 +02:00
parent 9481286169
commit 8752b36209
16 changed files with 831 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/store.ts
+119 -6
View File
@@ -169,10 +169,18 @@ export interface MediaRow {
download_error: string | null;
preview_error: string | null;
preview_attempts: number;
/** First message that carried this media; null on rows written before linking. */
message_id: string | null;
/** Embed title from Discord (YouTube and friends). */
title: string | null;
title_attempts: number;
title_error: string | null;
/** YouTube channel name from oEmbed, kept even if the video later disappears. */
title_author: string | null;
/** Last suspect (possibly throttled) preview failure; retried once, then left alone. */
preview_suspect: string | null;
/** Epoch ms when the suspect row may be retried. Null once confirmed unavailable. */
preview_retry_at: number | null;
/** Lowercased, punctuation-collapsed title for token-prefix search. */
title_key: string | null;
created_at: string;
@@ -242,6 +250,9 @@ CREATE TABLE IF NOT EXISTS media (
title_key TEXT,
title_attempts INTEGER NOT NULL DEFAULT 0,
title_error TEXT,
title_author TEXT,
preview_suspect TEXT,
preview_retry_at INTEGER,
created_at TEXT NOT NULL
);
@@ -295,6 +306,11 @@ export class ScrapeStore {
this.db.exec("ALTER TABLE media ADD COLUMN title_attempts INTEGER NOT NULL DEFAULT 0");
this.db.exec("ALTER TABLE media ADD COLUMN title_error TEXT");
}
if (!names.has("title_author")) this.db.exec("ALTER TABLE media ADD COLUMN title_author TEXT");
if (!names.has("preview_suspect")) {
this.db.exec("ALTER TABLE media ADD COLUMN preview_suspect TEXT");
this.db.exec("ALTER TABLE media ADD COLUMN preview_retry_at INTEGER");
}
this.migrateYouTubeShorts();
}
@@ -499,6 +515,88 @@ export class ScrapeStore {
* Rows that still need a cached preview. Attachment/sticker images and GIFs always;
* embed thumbnails only when `includeEmbeds` is set (download mode).
*/
/**
* Rows whose preview failed in a way that may be throttling rather than a dead link. Each is
* retried once after a delay; confirmed rows stay here and are skipped by normal passes, so a
* finished backfill never churns over genuinely unavailable media.
*/
dueSuspects(limit = 500, nowMs = Date.now()): MediaRow[] {
return this.db
.prepare(
`SELECT * FROM media
WHERE preview_hash IS NULL AND preview_error IS NULL
AND preview_suspect IS NOT NULL
AND preview_retry_at IS NOT NULL
AND preview_retry_at <= ?
ORDER BY preview_retry_at LIMIT ?`
)
.all(nowMs, limit) as unknown as MediaRow[];
}
/** Suspect rows that still have a retry coming. */
suspectCount(): number {
const row = this.db
.prepare(
"SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND preview_suspect IS NOT NULL AND preview_retry_at IS NOT NULL"
)
.get() as { count: number | null } | undefined;
return row?.count ?? 0;
}
/** Suspect rows whose single retry failed too; skipped by normal passes. */
unverifiedCount(): number {
const row = this.db
.prepare(
"SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND preview_suspect IS NOT NULL AND preview_retry_at IS NULL"
)
.get() as { count: number | null } | undefined;
return row?.count ?? 0;
}
/** Schedule the single delayed retry for a suspect row. */
markSuspect(id: number, error: string, retryAtMs: number): void {
this.db
.prepare("UPDATE media SET preview_suspect = ?, preview_retry_at = ? WHERE id = ?")
.run(error.slice(0, 200), retryAtMs, id);
}
/** The delayed retry failed too: keep the note but stop retrying. */
confirmSuspect(id: number, error: string): void {
this.db
.prepare("UPDATE media SET preview_suspect = ?, preview_retry_at = NULL WHERE id = ?")
.run(error.slice(0, 200), id);
}
/** The retry succeeded or the preview arrived another way. */
clearSuspect(id: number): void {
this.db.prepare("UPDATE media SET preview_suspect = NULL, preview_retry_at = NULL WHERE id = ?").run(id);
}
/** Discord-CDN rows closed out with an HTTP error; the recovery task retries these. */
discordPreviewFailures(limit = 5000): MediaRow[] {
return this.db
.prepare(
`SELECT * FROM media
WHERE preview_hash IS NULL AND preview_error IS NOT NULL
AND lower(url) LIKE '%discordapp%'
ORDER BY id LIMIT ?`
)
.all(limit) as unknown as MediaRow[];
}
clearPreviewError(id: number): void {
this.db.prepare("UPDATE media SET preview_error = NULL WHERE id = ?").run(id);
}
discordPreviewFailureCount(): number {
const row = this.db
.prepare(
"SELECT COUNT(*) AS count FROM media WHERE preview_hash IS NULL AND preview_error IS NOT NULL AND lower(url) LIKE '%discordapp%'"
)
.get() as { count: number | null } | undefined;
return row?.count ?? 0;
}
pendingPreviews(limit = 5000, includeEmbeds = false): MediaRow[] {
const local =
"((source IS NULL OR source != 'embed') AND kind IN ('image', 'gif', 'video', 'audio'))";
@@ -508,7 +606,10 @@ export class ScrapeStore {
return this.db
.prepare(
`SELECT * FROM media WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL ORDER BY id LIMIT ?`
`SELECT * FROM media
WHERE ${condition} AND preview_hash IS NULL AND preview_error IS NULL
AND preview_suspect IS NULL
ORDER BY id LIMIT ?`
)
.all(limit) as unknown as MediaRow[];
}
@@ -542,6 +643,10 @@ export class ScrapeStore {
titled: number;
/** Rows closed out without a title. */
titleFailed: number;
/** Preview failures kept for one delayed retry. */
suspect: number;
/** Suspect rows whose retry failed too. */
unverified: number;
byKind: Record<MediaKind, { total: number; pending: number; cached: number; failed: number }>;
} {
const empty = () => ({ total: 0, pending: 0, cached: 0, failed: 0 });
@@ -603,6 +708,8 @@ export class ScrapeStore {
titlePending: titles?.pending ?? 0,
titled: titles?.titled ?? 0,
titleFailed: titles?.failed ?? 0,
suspect: this.suspectCount(),
unverified: this.unverifiedCount(),
byKind
};
}
@@ -625,11 +732,13 @@ export class ScrapeStore {
.all(limit) as unknown as MediaRow[];
}
/** Store a resolved title and its search key. */
markTitle(id: number, title: string): void {
/** Store a resolved title, its search key, and the provider's channel name. */
markTitle(id: number, title: string, author?: string | null): void {
this.db
.prepare("UPDATE media SET title = ?, title_key = ?, title_error = NULL WHERE id = ?")
.run(title.slice(0, 300), normalizeTitleKey(title), id);
.prepare(
"UPDATE media SET title = ?, title_key = ?, title_error = NULL, title_author = COALESCE(?, title_author) WHERE id = ?"
)
.run(title.slice(0, 300), normalizeTitleKey(title), author ?? null, id);
}
/** Count one transient title failure and return the new total. */
@@ -873,7 +982,11 @@ export class ScrapeStore {
const { where, values } = this.mediaWhere(options);
const rows = this.db
.prepare(`SELECT * FROM media ${where} ORDER BY id DESC LIMIT ? OFFSET ?`)
.prepare(
`SELECT media.*,
(SELECT message_id FROM media_sources WHERE media_id = media.id LIMIT 1) AS message_id
FROM media ${where} ORDER BY id DESC LIMIT ? OFFSET ?`
)
.all(...values, limit, offset);
return rows as unknown as MediaRow[];