Backfill previews for paused scrapes and prefer richer dedupe rows
This commit is contained in:
Fisto
+107
-93
@@ -8,6 +8,7 @@
|
||||
import { rmSync } from "node:fs";
|
||||
import { isAbsolute, join } from "node:path";
|
||||
import {
|
||||
betterCandidate,
|
||||
dirSizeBytes,
|
||||
isNearLimit,
|
||||
planRetention,
|
||||
@@ -174,10 +175,28 @@ export class MediaService {
|
||||
download: {
|
||||
maxFileBytes: number(process.env.VESKSLOP_MEDIA_MAX_FILE_BYTES, 50 * 1024 * 1024),
|
||||
previewMaxBytes: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_BYTES, 5 * 1024 * 1024),
|
||||
previewMinGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MIN_MS, 200),
|
||||
previewMaxGapMs: number(process.env.VESKSLOP_MEDIA_PREVIEW_MAX_MS, 600),
|
||||
minGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MIN_MS, 1000),
|
||||
maxGapMs: number(process.env.VESKSLOP_MEDIA_DOWNLOAD_MAX_MS, 3000)
|
||||
}
|
||||
});
|
||||
|
||||
// Preview passes only run when a walk ends; fill in scrapes that paused early.
|
||||
void this.backfillPreviews();
|
||||
}
|
||||
|
||||
private async backfillPreviews(): Promise<void> {
|
||||
for (const meta of this.engine.list()) {
|
||||
if (meta.state === "running" || meta.state === "queued") continue;
|
||||
|
||||
try {
|
||||
const added = await this.engine.backfillPreviews(meta.id);
|
||||
if (added) this.deps.log?.(`backfilled ${added} previews for ${meta.id}`);
|
||||
} catch (error) {
|
||||
this.deps.log?.(`preview backfill failed for ${meta.id}: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Log slopbot in on first use. False when it is disabled or the login failed. */
|
||||
@@ -546,37 +565,8 @@ export class MediaService {
|
||||
after?: string;
|
||||
before?: string;
|
||||
} = {}): Record<MediaKind, { total: number; downloaded: number; cached: number; notDownloaded: number }> {
|
||||
// Deduplicate by URL across scrapes, newest scrape wins, like queryMedia.
|
||||
const byUrl = new Map<string, { kind: MediaKind; downloaded: boolean; cached: boolean }>();
|
||||
|
||||
for (const meta of this.engine.list()) {
|
||||
if (meta.state === "deleted") continue;
|
||||
if (options.channelId && meta.channelId !== options.channelId) continue;
|
||||
|
||||
const store = new ScrapeStore(scrapeDir(this.rootDir, meta.id));
|
||||
try {
|
||||
let storeOffset = 0;
|
||||
|
||||
for (;;) {
|
||||
const rows = store.queryMedia({ ...options, limit: 500, offset: storeOffset });
|
||||
if (!rows.length) break;
|
||||
storeOffset += rows.length;
|
||||
|
||||
for (const row of rows) {
|
||||
if (byUrl.has(row.url)) continue;
|
||||
byUrl.set(row.url, {
|
||||
kind: row.kind,
|
||||
downloaded: row.downloaded === 1,
|
||||
cached: row.preview_hash !== null
|
||||
});
|
||||
}
|
||||
|
||||
if (rows.length < 500) break;
|
||||
}
|
||||
} finally {
|
||||
store.close();
|
||||
}
|
||||
}
|
||||
// Same richest-per-URL dedupe as queryMedia.
|
||||
const candidates = this.collectCandidates(options);
|
||||
|
||||
const totals: Record<
|
||||
MediaKind,
|
||||
@@ -588,14 +578,14 @@ export class MediaService {
|
||||
audio: { total: 0, downloaded: 0, cached: 0, notDownloaded: 0 }
|
||||
};
|
||||
|
||||
for (const entry of byUrl.values()) {
|
||||
const bucket = totals[entry.kind];
|
||||
for (const { row } of candidates) {
|
||||
const bucket = totals[row.kind];
|
||||
bucket.total++;
|
||||
if (entry.downloaded) {
|
||||
if (row.downloaded === 1) {
|
||||
bucket.downloaded++;
|
||||
} else {
|
||||
bucket.notDownloaded++;
|
||||
if (entry.cached) bucket.cached++;
|
||||
if (row.preview_hash !== null) bucket.cached++;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -639,27 +629,23 @@ export class MediaService {
|
||||
}
|
||||
}
|
||||
|
||||
/** Read media rows across every scrape, newest scrape first. */
|
||||
queryMedia(
|
||||
options: {
|
||||
kind?: MediaKind;
|
||||
source?: string;
|
||||
scrapeId?: string;
|
||||
channelId?: string;
|
||||
authorId?: string;
|
||||
aspect?: string;
|
||||
resolution?: string;
|
||||
after?: string;
|
||||
before?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
} = {}
|
||||
): MediaItemResult[] {
|
||||
const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
|
||||
const offset = Math.max(0, options.offset ?? 0);
|
||||
const results: MediaItemResult[] = [];
|
||||
const seen = new Set<string>();
|
||||
let skipped = 0;
|
||||
/**
|
||||
* One entry per URL across every scrape. Scrapes are visited newest first, so a newer row
|
||||
* wins by default; an older row replaces it only when it is richer (downloaded, or with a
|
||||
* cached preview). Without this, a stopped or partial scrape hides usable previews.
|
||||
*/
|
||||
private collectCandidates(options: {
|
||||
kind?: MediaKind;
|
||||
source?: string;
|
||||
scrapeId?: string;
|
||||
channelId?: string;
|
||||
authorId?: string;
|
||||
aspect?: string;
|
||||
resolution?: string;
|
||||
after?: string;
|
||||
before?: string;
|
||||
}): Array<{ meta: ScrapeMeta; row: MediaRow }> {
|
||||
const candidates = new Map<string, { meta: ScrapeMeta; row: MediaRow }>();
|
||||
|
||||
for (const meta of this.engine.list()) {
|
||||
if (meta.state === "deleted") continue;
|
||||
@@ -686,53 +672,81 @@ export class MediaService {
|
||||
storeOffset += rows.length;
|
||||
|
||||
for (const row of rows) {
|
||||
// One card per URL, newest scrape wins.
|
||||
if (seen.has(row.url)) continue;
|
||||
seen.add(row.url);
|
||||
|
||||
if (skipped < offset) {
|
||||
skipped++;
|
||||
const current = candidates.get(row.url);
|
||||
if (!current) {
|
||||
candidates.set(row.url, { meta, row });
|
||||
continue;
|
||||
}
|
||||
if (results.length >= limit) break;
|
||||
|
||||
results.push({
|
||||
scrapeId: meta.id,
|
||||
itemId: row.id,
|
||||
channelId: meta.channelId,
|
||||
kind: row.kind,
|
||||
source: row.source,
|
||||
authorId: row.author_id,
|
||||
authorName: row.author_name,
|
||||
postedAt: row.posted_at,
|
||||
width: row.width,
|
||||
height: row.height,
|
||||
aspectRatio: row.aspect_ratio,
|
||||
contentType: row.content_type,
|
||||
durationSecs: row.duration_secs,
|
||||
url: row.url,
|
||||
playUrl: row.play_url,
|
||||
remotePreviewUrl: row.preview_url,
|
||||
localUrl:
|
||||
row.file_path && !meta.downloadDir
|
||||
? `/api/media/${meta.id}/${row.id}/file`
|
||||
: null,
|
||||
thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
|
||||
provider: row.provider,
|
||||
downloaded: row.downloaded === 1,
|
||||
createdAt: row.created_at
|
||||
});
|
||||
if (
|
||||
betterCandidate(
|
||||
{
|
||||
downloaded: current.row.downloaded === 1,
|
||||
hasPreview: current.row.preview_hash !== null
|
||||
},
|
||||
{ downloaded: row.downloaded === 1, hasPreview: row.preview_hash !== null }
|
||||
)
|
||||
) {
|
||||
candidates.set(row.url, { meta, row });
|
||||
}
|
||||
}
|
||||
|
||||
if (results.length >= limit || rows.length < 500) break;
|
||||
if (rows.length < 500) break;
|
||||
}
|
||||
} finally {
|
||||
store.close();
|
||||
}
|
||||
|
||||
if (results.length >= limit) break;
|
||||
}
|
||||
|
||||
return results;
|
||||
return [...candidates.values()];
|
||||
}
|
||||
|
||||
/** Read media rows across every scrape, richest per URL, newest scrape first. */
|
||||
queryMedia(
|
||||
options: {
|
||||
kind?: MediaKind;
|
||||
source?: string;
|
||||
scrapeId?: string;
|
||||
channelId?: string;
|
||||
authorId?: string;
|
||||
aspect?: string;
|
||||
resolution?: string;
|
||||
after?: string;
|
||||
before?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
} = {}
|
||||
): MediaItemResult[] {
|
||||
const limit = Math.max(1, Math.min(options.limit ?? 60, 500));
|
||||
const offset = Math.max(0, options.offset ?? 0);
|
||||
|
||||
return this.collectCandidates(options)
|
||||
.slice(offset, offset + limit)
|
||||
.map(({ meta, row }) => ({
|
||||
scrapeId: meta.id,
|
||||
itemId: row.id,
|
||||
channelId: meta.channelId,
|
||||
kind: row.kind,
|
||||
source: row.source,
|
||||
authorId: row.author_id,
|
||||
authorName: row.author_name,
|
||||
postedAt: row.posted_at,
|
||||
width: row.width,
|
||||
height: row.height,
|
||||
aspectRatio: row.aspect_ratio,
|
||||
contentType: row.content_type,
|
||||
durationSecs: row.duration_secs,
|
||||
url: row.url,
|
||||
playUrl: row.play_url,
|
||||
remotePreviewUrl: row.preview_url,
|
||||
localUrl:
|
||||
row.file_path && !meta.downloadDir
|
||||
? `/api/media/${meta.id}/${row.id}/file`
|
||||
: null,
|
||||
thumbUrl: row.preview_hash ? `/api/media/${meta.id}/${row.id}/thumb` : null,
|
||||
provider: row.provider,
|
||||
downloaded: row.downloaded === 1,
|
||||
createdAt: row.created_at
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user