diff --git a/CURRENT.md b/CURRENT.md index ec4c990..e74ecb7 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,14 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C61 Media browsing performance. The grid's merge step rescanned every scrape database per + request (about 300 ms for a page, 250 ms for stats, both refetched on every scroll append). + `MediaService` now keeps a candidate cache keyed by a scrape revision plus the normalized + filter shape (paging and empties dropped): one scan per filter set, then ~5 ms per page, and + stats share the scan. The client skips stats when the filters are unchanged (60 s TTL), updates + only the pager instead of rebuilding the toolbar on page loads, and renders tiles with + `content-visibility`/`decoding=async`/low fetch priority. Measured: 302 ms first page, 5 ms + later pages; smoke test streamed 2,480 tiles with no console errors. - [x] C60 Incremental re-scrapes and archive cleanup. A new scrape of a channel that already has a completed walk now starts after that walk's newest message (the best completed scrape by message count), so it only covers the newer gap instead of re-fetching history; the previous diff --git a/docs/media-scraper.md b/docs/media-scraper.md index 1c05a39..d272d8c 100644 --- a/docs/media-scraper.md +++ b/docs/media-scraper.md @@ -190,6 +190,17 @@ Previews run after every walk: of 25 with a 60 s cooldown between them, because bursts of thumbnail fetches trigger Discord and provider rate limits. A batch counts rows, not successes. +## Browsing performance + +Every grid request used to rescan every scrape database to merge URLs. `MediaService` now keeps a +short-lived candidate cache: the key is a revision of the scrape metadata (id, state, update time, +preview count) plus the normalized filter shape — page limits, offsets, and empty values are +dropped, so every page of one filter set shares a single scan and later pages return in +milliseconds. The 15 s TTL and the revision key cover new pages, finished scrapes, and freshly +cached previews. The client only refetches stats when the filters change (or after a minute), and +grid cards use lazy async decoding with CSS `content-visibility` so a long stream does not pay +layout and paint for off-screen tiles. + ## Rate limits Discord and the provider APIs throttle bursts. The engine detects limiting and holds all media diff --git a/poc/public/media.css b/poc/public/media.css index f9b7ac5..b8c86f6 100644 --- a/poc/public/media.css +++ b/poc/public/media.css @@ -78,6 +78,12 @@ text-decoration: none; } +/* Skip layout and paint for cards far off screen: the DOM can grow to thousands. */ +.vk-media-figure { + content-visibility: auto; + contain-intrinsic-size: auto 170px; +} + .vk-media-figure { display: flex; flex-direction: column; diff --git a/poc/public/media.js b/poc/public/media.js index 1396030..92f49c4 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -359,14 +359,23 @@ ); } - async function loadStats() { + async function loadStats(force = false) { + const key = filterQuery(); + const fresh = state.statsAt && Date.now() - state.statsAt < 60_000; + if (!force && fresh && state.stats && state.statsKey === key) { + renderStatus(); + return; + } + try { - state.stats = await window.VK.api.get(`/api/media/stats?${filterQuery()}`); + state.stats = await window.VK.api.get(`/api/media/stats?${key}`); + state.statsKey = key; + state.statsAt = Date.now(); } catch { state.stats = null; } renderStatus(); - if (state.pageMode === "page" && state.view === "media") renderToolbar(); + updatePager(); } /** Append only the cards that are new, so existing images never get rebuilt. */ @@ -444,13 +453,13 @@ } layoutBuffer(); - void loadStats(); + void loadStats(reset); if (state.pageMode === "infinite") { void prefetchNext(); void fillBuffer(); } else { state.pageCooldownUntil = Date.now() + 2000; - renderToolbar(); + updatePager(); void prefetchNext(); } } @@ -1307,6 +1316,8 @@ if (imageSource) { const img = document.createElement("img"); img.loading = "lazy"; + img.decoding = "async"; + img.fetchPriority = "low"; img.src = imageSource; img.alt = item.kind; img.addEventListener("error", () => placeholder(media, item), { once: true }); @@ -1389,9 +1400,17 @@ } /** Dropdown with the metadata-field checkboxes. */ + /** Re-render only the pager after a page load, not the whole toolbar. */ + function updatePager() { + if (state.pageMode !== "page") return; + if (!state.pagerHost?.isConnected) return; + state.pagerHost.replaceWith(pageControls()); + } + /** Infinite/page flippy toggle plus the pager shown in page mode. */ function pageControls() { const wrap = el("span", "vk-media-page-controls"); + state.pagerHost = wrap; const toggle = el("span", "vk-media-toggle"); for (const [mode, label] of [ @@ -1514,6 +1533,8 @@ function applyDraft() { state.page = 1; state.pagePrefetch = null; + state.statsKey = null; + state.statsAt = 0; savePage(); state.kind = state.draft.kind; state.filters = { @@ -2035,6 +2056,8 @@ state.page = 1; state.pageCooldownUntil = 0; state.pagePrefetch = null; + state.statsKey = null; + state.statsAt = 0; savePage(); state.prefetched = null; state.items = []; diff --git a/poc/src/media.ts b/poc/src/media.ts index 1335b35..7716346 100644 --- a/poc/src/media.ts +++ b/poc/src/media.ts @@ -191,6 +191,13 @@ export class MediaService { /** The newest bot and scrape errors, capped. Surfaced in the Bot tab. */ private readonly errors: MediaErrorEntry[] = []; + /** + * Merged-candidate cache. Every grid request otherwise rescans every scrape database; the + * key includes a revision of the scrape metadata and preview counters, so new pages, state + * changes, and fresh previews invalidate it. The TTL catches anything the revision misses. + */ + private readonly candidateCache = new Map }>(); + /** One-shot maintenance: re-fetch previews for Discord rows closed out with HTTP errors. */ private readonly recover = { state: "idle" as "idle" | "running" | "paused" | "done", @@ -1168,6 +1175,14 @@ export class MediaService { * wins by default; an older row replaces it only when it is richer (downloaded, or with a * cached preview). Without this, a stopped or partial scrape hides usable previews. */ + /** Cheap change signal for the candidate cache: ids, states, update times, preview counts. */ + private catalogRevision(): string { + return this.engine + .list() + .map(meta => `${meta.id}:${meta.state}:${meta.updatedAt}:${meta.previewed ?? 0}`) + .join("|"); + } + private collectCandidates(options: { kind?: MediaKind; source?: string; @@ -1195,6 +1210,54 @@ export class MediaService { after?: string; before?: string; sort?: MediaSort; + }): Array<{ meta: ScrapeMeta; row: MediaRow }> { + // Drop paging and empty values: every page of one filter set shares a single scan. + const shape: Record = {}; + for (const [name, value] of Object.entries(options)) { + if (name === "limit" || name === "offset") continue; + if (value === undefined || value === null || value === "") continue; + if (Array.isArray(value) && value.length === 0) continue; + shape[name] = value; + } + + const key = `${this.catalogRevision()}\u0000${JSON.stringify(shape)}`; + const hit = this.candidateCache.get(key); + const now = Date.now(); + if (hit && now - hit.at < 15_000) return hit.value; + + const value = this.collectCandidatesUncached(options); + if (this.candidateCache.size > 60) this.candidateCache.clear(); + this.candidateCache.set(key, { at: now, value }); + return value; + } + + private collectCandidatesUncached(options: { + kind?: MediaKind; + source?: string; + scrapeId?: string; + channelId?: string; + authorId?: string; + authorIds?: string[]; + origin?: "dm" | "guild"; + cachedOnly?: boolean; + exts?: string[]; + excludeHosts?: string[]; + excludeExts?: string[]; + excludeProviders?: string[]; + excludeSources?: string[]; + includeProviders?: string[]; + includeSources?: string[]; + titleQuery?: string; + titleExact?: boolean; + sizeLessThan?: number; + sizeGreaterThan?: number; + lengthLessThan?: number; + lengthGreaterThan?: number; + aspect?: string; + resolution?: string; + after?: string; + before?: string; + sort?: MediaSort; }): Array<{ meta: ScrapeMeta; row: MediaRow }> { const candidates = new Map();