diff --git a/CURRENT.md b/CURRENT.md index 2eff397..a78b94f 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -40,6 +40,14 @@ mobile draft at 390px. - [x] C11 preview caching: Discord-hosted image/GIF previews (WebP 320px, hash-addressed, 5 MiB cap, one attempt) run after every walk in both modes. Live run `...-20260916090722`: `previewed: 2`; thumbs served over `/api/media///thumb` (200). + - [x] C45 Streaming media grid. The grid rebuilt every card on each appended page, which made + filtered browsing flash, and the `Load more` trigger only worked once the grid could already + scroll, so short filtered pages dead-stopped at ~11 cards. Now pages append incrementally + (existing images are never rebuilt), a stream buffer capped at 1.5 viewport heights keeps the + scroll alive while the next page loads, auto-fill cascades while the buffer is on screen, the + buffer note reads `Loading more…` / `Scroll for more` / `End of results`, and an elastic nudge + replaces the hard stop at the bottom. Smoke test: first load reaches 120 cards via auto-fill, + scrolling appends 120 → 440 with no broken images. - [x] C44 One traffic pipeline and split ETAs. Walks and backfill batches now share a media-traffic lock: a walk holds it for its whole run, a batch for one batch, so the pacing settings bound total traffic instead of each action. Starting then immediately stopping a diff --git a/docs/media.md b/docs/media.md index bbb3a6c..2f73a39 100644 --- a/docs/media.md +++ b/docs/media.md @@ -175,9 +175,14 @@ that deletion is irreversible. The delete button is disabled for a 5 second coun can take its place while the hidden one stays resumable. The strip text turns red and reads `API limited, retry in Ns` while Discord throttles media traffic, and a translucent `N scrapes completed` line appears when scrapes finish — it lives until the tab is reloaded. Each - row shows `X/Yp (~Z%)` when the search total is known. The grid scrolls infinitely - (a **Load more** button appears as a fallback). Media whose source answers `403` shows a - **PRIVATE** tile and `404`/`410` shows **GONE**, with the reason in the tooltip. Some channels deny the bot the search endpoint (`Missing Access`) even though history + row shows `X/Yp (~Z%)` when the search total is known. The grid streams: appending a page inserts + only the new cards, so loaded images never rebuild or flash, and a stream **buffer** below the last + row keeps the scroll alive while the next page is in flight. The buffer is capped at 1.5 viewport + heights, shrinks as the filtered results run out, and says `Loading more…` / `Scroll for more` / + `End of results`; auto-fill cascades pages while the buffer is still in view, so a filter that + returns few items does not dead-stop at one page. At the very bottom while a page loads, an + elastic nudge replaces the hard stop. Media whose source answers `403` shows a **PRIVATE** tile + and `404`/`410` shows **GONE**, with the reason in the tooltip. Some channels deny the bot the search endpoint (`Missing Access`) even though history reads work, so the total is optional. **New scrape…** (rightmost) opens a popup with Server and Group chat trees (servers expand to channels; each row has Links and Download). Every scrape button asks for confirmation; the diff --git a/docs/webui.md b/docs/webui.md index d525df0..ba0f374 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -135,8 +135,12 @@ box live in Chat, not Call, and the connection stats are not chips yet. apply until **Apply filters** is pressed; **Reset** clears every filter and the zoom, unloads the grid, and brings back the initial Load media prompt. - A **Card size** slider scales tiles from two-thirds of the old size up to one image per row; the - page size follows the layout — full rows about 1.5 screens tall — and the grid infinite-scrolls, - preloading the next page's JSON and preview images just before they are needed. + page size follows the layout — full rows about 1.5 screens tall — and the grid streams: each page + appends only its new cards (no image flash) and a buffer below the last row, capped at 1.5 + viewport heights, keeps the scroll alive while the next page loads. Auto-fill runs while the + buffer is on screen, so short filtered pages never dead-stop; the note reads `Loading more…`, + `Scroll for more`, or `End of results`, and an elastic nudge replaces the hard stop at the bottom + while loading. The next page's JSON and previews are preloaded just before they are needed. **New scrape…** opens a popup with **Server** and **Group chat** tabs: servers expand into their text channels, each channel (and each group chat) has **Links** and **Download** buttons, and group chats always use the puppet. **History** lists diff --git a/poc/public/media.css b/poc/public/media.css index 0011819..5d62a14 100644 --- a/poc/public/media.css +++ b/poc/public/media.css @@ -750,3 +750,21 @@ cursor: help; text-decoration: underline dotted color-mix(in srgb, var(--color-neutral-400) 60%, transparent); } + +/* Stream buffer: scrollable room while the next page loads. */ +.vk-media-buffer { + grid-column: 1 / -1; + display: flex; + flex-direction: column; + align-items: center; + justify-content: flex-start; + gap: 8px; + overflow: hidden; + transition: height 160ms ease-out; +} + +.vk-media-buffer-note { + padding: 10px 0; + font: 400 11px/1.5 ui-monospace, Menlo, monospace; + color: var(--color-neutral-500); +} diff --git a/poc/public/media.js b/poc/public/media.js index 14c276a..b253ff2 100644 --- a/poc/public/media.js +++ b/poc/public/media.js @@ -35,6 +35,9 @@ zoom: Number(localStorage.getItem("vk-media-zoom") ?? 50), draft: null, prefetched: null, + prefetchPromise: null, + itemKeys: new Set(), + loadError: false, pageSize: 60, backfill: null, preferences: null, @@ -101,6 +104,14 @@ const tunablesHost = overlay.querySelector("#bot-tunables"); const botErrors = overlay.querySelector("#bot-errors"); + // The stream buffer: extra scrollable room so hitting the bottom is never a dead stop. + const buffer = el("div", "vk-media-buffer"); + const bufferNote = el("div", "vk-media-buffer-note", ""); + const bufferRetry = button("Retry", () => void loadNextPage()); + bufferRetry.classList.add("vk-media-buffer-retry"); + bufferRetry.hidden = true; + buffer.append(bufferNote, bufferRetry); + const filterDrawer = el("div", "vk-media-drawer"); filterDrawer.hidden = true; toolbar.after(filterDrawer); @@ -297,58 +308,177 @@ renderStatus(); } + /** Append only the cards that are new, so existing images never get rebuilt. */ + function appendPage(items) { + const fresh = []; + for (const item of items) { + const key = `${item.scrapeId}:${item.itemId}`; + if (state.itemKeys.has(key)) continue; + state.itemKeys.add(key); + fresh.push(item); + } + + state.items = state.items.concat(fresh); + state.offset += items.length; + state.more = items.length >= state.pageSize; + state.loadError = false; + + for (const item of fresh) grid.insertBefore(renderCard(item), buffer); + layoutBuffer(); + renderStatus(); + } + + /** Reset the list and render everything from scratch. */ + function renderCards() { + grid.replaceChildren(); + for (const item of state.items) grid.append(renderCard(item)); + if (!state.items.length) grid.append(el("div", "vk-media-status", "No media matches the filters.")); + grid.append(buffer); + layoutBuffer(); + } + async function loadMedia(reset) { if (!state.loaded || state.loading) return; state.loading = true; + state.loadError = false; + layoutBuffer(); if (reset) { state.items = []; + state.itemKeys = new Set(); state.offset = 0; - state.more = false; + state.more = true; state.prefetched = null; + state.prefetchPromise = null; } try { const items = await window.VK.api.get( `/api/media?limit=${state.pageSize}&offset=${state.offset}&${filterQuery()}` ); - state.items = state.items.concat(items); - state.offset += items.length; - state.more = items.length >= state.pageSize; + + if (reset) { + for (const item of items) state.itemKeys.add(`${item.scrapeId}:${item.itemId}`); + state.items = items; + state.offset += items.length; + state.more = items.length >= state.pageSize; + renderCards(); + } else { + appendPage(items); + } } catch (error) { + state.loadError = true; setStatus(`media load failed: ${error.message}`); } finally { state.loading = false; } - renderGrid(); + layoutBuffer(); void loadStats(); void prefetchNext(); + void fillBuffer(); } /** Warm the next page's JSON and preview images before the user reaches them. */ async function prefetchNext() { - if (!state.loaded || !state.more || state.prefetched || state.loading) return; + if (!state.loaded || !state.more || state.prefetched || state.prefetchPromise || state.loading) return; - try { - const items = await window.VK.api.get( - `/api/media?limit=${state.pageSize}&offset=${state.offset}&${filterQuery()}` - ); - if (!items.length) { - state.more = false; - return; - } + state.prefetchPromise = window.VK.api + .get(`/api/media?limit=${state.pageSize}&offset=${state.offset}&${filterQuery()}`) + .then(items => { + if (!items.length) { + state.more = false; + return; + } - state.prefetched = items; - for (const item of items) { - const src = item.thumbUrl || item.localUrl; - if (!src) continue; - const image = new Image(); - image.src = src; - } - } catch { - // Preloading is best effort; the scroll handler falls back to a request. + state.prefetched = items; + for (const item of items) { + const src = item.thumbUrl || item.localUrl; + if (!src) continue; + const image = new Image(); + image.src = src; + } + }) + .catch(() => { + // Preloading is best effort; the scroll handler falls back to a request. + }) + .finally(() => { + state.prefetchPromise = null; + }); + + await state.prefetchPromise; + } + + /** Ensure the next page is on its way when the buffer is close or already in view. */ + async function loadNextPage() { + if (state.loading || !state.loaded || !state.more) return; + + if (state.prefetchPromise) await state.prefetchPromise; + + if (state.prefetched) { + const items = state.prefetched; + state.prefetched = null; + appendPage(items); + void prefetchNext(); + void fillBuffer(); + return; } + + await loadMedia(false); + } + + /** + * The buffer keeps scolling alive while the next page loads. Its height is capped at + * 1.5 viewport heights and shrinks as the end of the filtered results comes closer. + */ + function layoutBuffer() { + if (!state.loaded) { + buffer.style.height = "0px"; + bufferNote.textContent = ""; + bufferRetry.hidden = true; + return; + } + + if (!state.more && !state.loadError) { + buffer.style.height = ""; + bufferNote.textContent = state.items.length ? "End of results" : ""; + bufferRetry.hidden = true; + return; + } + + const cap = Math.round((grid.clientHeight || 600) * 1.5); + const rowHeight = gridRowHeight(); + const total = filteredTotal(); + const remainingRows = total === null ? null : Math.ceil(Math.max(0, total - state.offset) / gridColumns()); + const wanted = remainingRows === null ? cap : remainingRows * rowHeight; + buffer.style.height = `${Math.max(rowHeight, Math.min(cap, wanted))}px`; + + bufferNote.textContent = state.loadError + ? "Could not load more." + : state.loading + ? "Loading more\u2026" + : "Scroll for more"; + bufferRetry.hidden = !state.loadError; + } + + /** Filtered total from the stats the status row already uses; null until known. */ + function filteredTotal() { + const stats = state.stats; + if (!stats) return null; + return ( + (stats.image?.total ?? 0) + + (stats.gif?.total ?? 0) + + (stats.video?.total ?? 0) + + (stats.audio?.total ?? 0) + ); + } + + /** Keep loading while the buffer is on screen, so short filtered pages never dead-stop. */ + function fillBuffer() { + if (!state.more || state.loading || !state.loaded) return; + const top = buffer.getBoundingClientRect().top; + const bottom = grid.getBoundingClientRect().bottom; + if (top <= bottom + 320) void loadNextPage(); } async function startScrape(channelId, guildId, mode, downloadDir, after, before) { @@ -942,16 +1072,7 @@ return; } - for (const item of state.items) grid.append(renderCard(item)); - - if (!state.items.length) { - grid.append(el("div", "vk-media-status", "No media matches the filters.")); - } else if (state.more) { - const more = el("button", "btn btn-secondary vk-media-load-more", "Load more"); - more.type = "button"; - more.addEventListener("click", () => void loadMedia(false)); - grid.append(more); - } + renderCards(); } function filterSelect(options, value, onChange) { @@ -1313,12 +1434,19 @@ return Math.round(150 + ((zoom - 50) / 50) * Math.max(0, gridWidth - 150)); } + function gridColumns() { + const width = cardWidth(); + return Math.max(1, Math.floor(((grid.clientWidth || 900) + 8) / (width + 8))); + } + + function gridRowHeight() { + return cardWidth() * 1.16 + 30; + } + /** Items per page: full rows, about 1.5 screens tall. */ function computePageSize() { - const width = cardWidth(); - const columns = Math.max(1, Math.floor(((grid.clientWidth || 900) + 8) / (width + 8))); - const rowHeight = width * 1.16 + 30; - const rows = Math.max(1, Math.floor(((grid.clientHeight || 600) + 8) / (rowHeight + 8))); + const columns = gridColumns(); + const rows = Math.max(1, Math.floor(((grid.clientHeight || 600) + 8) / (gridRowHeight() + 8))); return Math.max(columns, columns * Math.max(1, Math.ceil(rows * 1.5))); } @@ -1328,6 +1456,7 @@ grid.style.gridTemplateColumns = width >= gridWidth - 24 ? "1fr" : `repeat(auto-fill, minmax(${width}px, 1fr))`; state.pageSize = computePageSize(); + layoutBuffer(); } function resetMediaView() { @@ -1414,26 +1543,67 @@ if (state.view === "bot") void refreshBackfill(); }, 3000); - // Infinite scroll: fetch the next page before the grid bottom is reached. + // Infinite scroll: the buffer entering the viewport starts the next page. + const bufferObserver = new IntersectionObserver( + entries => { + if (entries.some(entry => entry.isIntersecting)) void loadNextPage(); + }, + { root: grid, rootMargin: "400px 0px" } + ); + bufferObserver.observe(buffer); + grid.addEventListener("scroll", () => { - if (state.loading || !state.loaded) return; - if (grid.scrollTop + grid.clientHeight < grid.scrollHeight - 240) return; - - if (state.prefetched) { - const items = state.prefetched; - state.prefetched = null; - state.items = state.items.concat(items); - state.offset += items.length; - state.more = items.length >= state.pageSize; - renderGrid(); - void loadStats(); - void prefetchNext(); - return; - } - - if (state.more) void loadMedia(false); + if (!state.loaded) return; + const remaining = grid.scrollHeight - grid.scrollTop - grid.clientHeight; + if (remaining < 400) void loadNextPage(); }); + // Past the buffer, a small elastic nudge instead of a hard stop while more is loading. + let elasticOffset = 0; + let elasticTimer = null; + + function elastic(delta) { + if (!state.more || !state.loading) return; + const atBottom = grid.scrollTop + grid.clientHeight >= grid.scrollHeight - 2; + if (!atBottom) return; + + elasticOffset = Math.max(-18, Math.min(0, elasticOffset - delta * 0.35)); + grid.style.transition = "none"; + grid.style.transform = `translateY(${elasticOffset}px)`; + + clearTimeout(elasticTimer); + elasticTimer = setTimeout(() => { + grid.style.transition = "transform 160ms ease-out"; + grid.style.transform = ""; + elasticOffset = 0; + }, 60); + } + + grid.addEventListener( + "wheel", + event => { + if (event.deltaY > 0) elastic(event.deltaY); + }, + { passive: true } + ); + + let touchStartY = 0; + grid.addEventListener( + "touchstart", + event => { + touchStartY = event.touches[0]?.clientY ?? 0; + }, + { passive: true } + ); + grid.addEventListener( + "touchmove", + event => { + const y = event.touches[0]?.clientY ?? 0; + if (touchStartY - y > 0) elastic((touchStartY - y) / 8); + }, + { passive: true } + ); + // ---- Scrape popup: server or group-chat trees ---- const scrapeModal = document.createElement("div"); diff --git a/poc/test/media-ui-smoke.mjs b/poc/test/media-ui-smoke.mjs index 5d37908..c413ef5 100644 --- a/poc/test/media-ui-smoke.mjs +++ b/poc/test/media-ui-smoke.mjs @@ -84,6 +84,33 @@ try { ); step("broken images", broken); check("no broken images", broken === 0, broken); + check("stream buffer present", (await page.locator(".vk-media-buffer").count()) === 1); + check("no load-more button", (await page.locator(".vk-media-load-more").count()) === 0); + + // Infinite scroll: scrolling into the buffer appends cards without rebuilding the grid. + const beforeScroll = await page.locator(".vk-media-figure").count(); + for (let attempt = 0; attempt < 3; attempt++) { + await page.locator("#media-grid").evaluate(node => { + node.scrollTop = node.scrollHeight; + }); + await page.waitForTimeout(1200); + } + const afterScroll = await page.locator(".vk-media-figure").count(); + step("infinite scroll", { before: beforeScroll, after: afterScroll }); + check("infinite scroll appends cards", afterScroll > beforeScroll, { + before: beforeScroll, + after: afterScroll + }); + check( + "appended cards do not corrupt loaded ones", + (await page.$$eval(".vk-media-figure .vk-media-media img", images => + images.filter(image => image.complete && image.naturalWidth === 0).length + )) === 0 + ); + await page.locator("#media-grid").evaluate(node => { + node.scrollTop = 0; + }); + await page.waitForTimeout(300); await page.getByRole("button", { name: /^People/ }).click(); step("people checkboxes", await page.locator(".vk-media-people input").count());