zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Attempt the channel lookup once per titled row

Browse Source
This commit is contained in:
Fisto
2026-09-20 19:53:22 +02:00
parent cc3d62336c
commit 19bf6e16fc
2 changed files with 21 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media-scrape/src/store.ts
+13 -4
View File
@@ -251,6 +251,7 @@ CREATE TABLE IF NOT EXISTS media (
title_attempts INTEGER NOT NULL DEFAULT 0,
title_error TEXT,
title_author TEXT,
title_author_checked INTEGER NOT NULL DEFAULT 0,
preview_suspect TEXT,
preview_retry_at INTEGER,
created_at TEXT NOT NULL
@@ -307,6 +308,9 @@ export class ScrapeStore {
this.db.exec("ALTER TABLE media ADD COLUMN title_error TEXT");
}
if (!names.has("title_author")) this.db.exec("ALTER TABLE media ADD COLUMN title_author TEXT");
if (!names.has("title_author_checked")) {
this.db.exec("ALTER TABLE media ADD COLUMN title_author_checked INTEGER NOT NULL DEFAULT 0");
}
if (!names.has("preview_suspect")) {
this.db.exec("ALTER TABLE media ADD COLUMN preview_suspect TEXT");
this.db.exec("ALTER TABLE media ADD COLUMN preview_retry_at INTEGER");
@@ -715,7 +719,8 @@ export class ScrapeStore {
const titles = this.db
.prepare(
`SELECT
SUM(title IS NULL AND title_error IS NULL
SUM(title_error IS NULL
AND (title IS NULL OR (title_author IS NULL AND title_author_checked = 0))
AND (lower(COALESCE(provider, '')) LIKE '%youtube%'
OR lower(url) LIKE '%youtube.com/%'
OR lower(url) LIKE '%youtu.be/%')) AS pending,
@@ -742,10 +747,12 @@ export class ScrapeStore {
* provider oEmbed in the backfill queue.
*/
pendingTitles(limit = 500): MediaRow[] {
// Missing title, or a title whose channel was never fetched (one attempt per row).
return this.db
.prepare(
`SELECT * FROM media
WHERE title IS NULL AND title_error IS NULL
WHERE title_error IS NULL
AND (title IS NULL OR (title_author IS NULL AND title_author_checked = 0))
AND (lower(COALESCE(provider, '')) LIKE '%youtube%'
OR lower(url) LIKE '%youtube.com/%'
OR lower(url) LIKE '%youtu.be/%')
@@ -758,7 +765,7 @@ export class ScrapeStore {
markTitle(id: number, title: string, author?: string | null): void {
this.db
.prepare(
"UPDATE media SET title = ?, title_key = ?, title_error = NULL, title_author = COALESCE(?, title_author) WHERE id = ?"
"UPDATE media SET title = ?, title_key = ?, title_error = NULL, title_author = COALESCE(?, title_author), title_author_checked = 1 WHERE id = ?"
)
.run(title.slice(0, 300), normalizeTitleKey(title), author ?? null, id);
}
@@ -774,7 +781,9 @@ export class ScrapeStore {
}
markTitleError(id: number, error: string): void {
this.db.prepare("UPDATE media SET title_error = ? WHERE id = ?").run(error.slice(0, 200), id);
this.db
.prepare("UPDATE media SET title_error = ?, title_author_checked = 1 WHERE id = ?")
.run(error.slice(0, 200), id);
}
/** Count one transient preview failure and return the new total. */
media-scrape/test/run.ts
+8 -1
View File
@@ -1596,18 +1596,25 @@ void (async () => {
]);
const label = (url: string) => url.split("=")[1] ?? url.split("/").pop();
// A titled row is still pending once for its channel (author) lookup.
const pending = store.pendingTitles(10).map(row => label(row.url));
assert.deepEqual(pending.sort(), ["notitle", "short"].sort());
assert.deepEqual(pending.sort(), ["notitle", "short", "withtitle"].sort());
const target = store.pendingTitles(10).find(row => row.url.includes("notitle"));
const dead = store.pendingTitles(10).find(row => row.url.includes("youtu.be"));
const titled = store.pendingTitles(10).find(row => row.url.includes("withtitle"));
assert.ok(target);
assert.ok(dead);
assert.ok(titled);
store.markTitle(target.id, "Resolved Title");
assert.equal(store.getMedia(target.id)?.title, "Resolved Title");
assert.equal(store.pendingTitles(10).some(row => row.id === target.id), false);
store.markTitle(titled.id, "Has a title", "Some Channel");
assert.equal(store.getMedia(titled.id)?.title_author, "Some Channel");
assert.equal(store.pendingTitles(10).some(row => row.id === titled.id), false);
store.markTitleError(dead.id, "no title after 1 attempt(s)");
assert.equal(store.pendingTitles(10).some(row => row.id === dead.id), false);
assert.equal(store.previewCounts().titlePending, 0);