zib/Veskslop-t
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add sort and exclusion filters to the media grid

Browse Source
This commit is contained in:
Fisto
2026-09-17 11:01:50 +02:00
parent cc14140f34
commit 515af01e84
13 changed files with 561 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
poc/public/media.css
+23
View File
@@ -768,3 +768,26 @@
font: 400 11px/1.5 ui-monospace, Menlo, monospace;
color: var(--color-neutral-500);
}
/* Advanced drawer: the "Do not include" group. */
.vk-media-exclude-title {
cursor: pointer;
font: 600 11px/1.6 ui-monospace, Menlo, monospace;
color: var(--color-neutral-300);
}
.vk-media-exclude-group {
margin-left: 12px;
}
.vk-media-exclude-group summary {
cursor: pointer;
}
.vk-media-small-input {
width: 90px;
}
.vk-chip-strike {
text-decoration: line-through;
}
poc/public/media.js
+216 -3
View File
@@ -251,6 +251,12 @@
renderScrapes();
}
const SIZE_UNITS = { B: 1, KB: 1024, MB: 1024 ** 2, GB: 1024 ** 3 };
function sizeUnit(unit) {
return SIZE_UNITS[unit] ?? 1;
}
function filterQuery() {
const kind = state.kind === "all" ? "" : `kind=${state.kind}&`;
const source = state.filters.source ? `source=${state.filters.source}&` : "";
@@ -264,11 +270,34 @@
const excludeHosts = state.filters.excludeHosts.length
? `excludeHosts=${state.filters.excludeHosts.join(",")}&`
: "";
const sort = state.filters.sort && state.filters.sort !== "newest" ? `sort=${state.filters.sort}&` : "";
const excludeProviders = state.filters.excludeProviders.length
? `excludeProviders=${state.filters.excludeProviders.join(",")}&`
: "";
const excludeSources = state.filters.excludeSources.length
? `excludeSources=${state.filters.excludeSources.join(",")}&`
: "";
const excludeExts = state.filters.excludeExts.length
? `excludeExt=${state.filters.excludeExts.join(",")}&`
: "";
const amount = (value, unit) => {
const number = Number(value);
if (value === "" || !Number.isFinite(number) || number < 0) return "";
return String(Math.round(number * (unit ? sizeUnit(unit) : 1)));
};
const sizeLt = state.filters.sizeLt !== "" ? `sizeLt=${amount(state.filters.sizeLt, state.filters.sizeLtUnit)}&` : "";
const sizeGt = state.filters.sizeGt !== "" ? `sizeGt=${amount(state.filters.sizeGt, state.filters.sizeGtUnit)}&` : "";
const lenLt = state.filters.lenLt !== "" ? `lenLt=${amount(state.filters.lenLt)}&` : "";
const lenGt = state.filters.lenGt !== "" ? `lenGt=${amount(state.filters.lenGt)}&` : "";
const aspect = state.filters.aspect ? `aspect=${state.filters.aspect}&` : "";
const resolution = state.filters.resolution ? `resolution=${state.filters.resolution}&` : "";
const after = state.filters.after ? `after=${state.filters.after}T00:00:00.000Z&` : "";
const before = state.filters.before ? `before=${state.filters.before}T23:59:59.999Z&` : "";
return `${kind}${source}${channel}${authors}${origin}${cached}${exts}${excludeHosts}${aspect}${resolution}${after}${before}`;
return (
`${kind}${source}${channel}${authors}${origin}${cached}${exts}${excludeHosts}` +
`${sort}${excludeProviders}${excludeSources}${excludeExts}${sizeLt}${sizeGt}${lenLt}${lenGt}` +
`${aspect}${resolution}${after}${before}`
);
}
function kindTotals(field) {
@@ -1137,6 +1166,16 @@
cachedOnly: false,
exts: [],
excludeHosts: ["4chan.org", "i.4cdn.org"],
sort: "newest",
excludeProviders: [],
excludeSources: [],
excludeExts: [],
sizeLt: "",
sizeGt: "",
sizeLtUnit: "MB",
sizeGtUnit: "MB",
lenLt: "",
lenGt: "",
aspect: "",
resolution: "",
after: "",
@@ -1146,7 +1185,16 @@
function ensureDraft() {
if (!state.draft) {
state.draft = { kind: state.kind, ...state.filters, authorIds: [...state.filters.authorIds] };
state.draft = {
...state.filters,
kind: state.kind,
authorIds: [...state.filters.authorIds],
exts: [...state.filters.exts],
excludeHosts: [...state.filters.excludeHosts],
excludeProviders: [...state.filters.excludeProviders],
excludeSources: [...state.filters.excludeSources],
excludeExts: [...state.filters.excludeExts]
};
}
}
@@ -1160,6 +1208,16 @@
cachedOnly: state.draft.cachedOnly,
exts: [...state.draft.exts],
excludeHosts: [...state.draft.excludeHosts],
sort: state.draft.sort,
excludeProviders: [...state.draft.excludeProviders],
excludeSources: [...state.draft.excludeSources],
excludeExts: [...state.draft.excludeExts],
sizeLt: state.draft.sizeLt,
sizeGt: state.draft.sizeGt,
sizeLtUnit: state.draft.sizeLtUnit,
sizeGtUnit: state.draft.sizeGtUnit,
lenLt: state.draft.lenLt,
lenGt: state.draft.lenGt,
aspect: state.draft.aspect,
resolution: state.draft.resolution,
after: state.draft.after,
@@ -1291,6 +1349,19 @@
const filters = section("Filters");
filters.append(
filterSelect(
[
["newest", "Newest first"],
["oldest", "Oldest first"],
["user", "User (A\u2192Z)"],
["size", "Largest size"],
["length", "Longest length"]
],
draft.sort,
value => {
draft.sort = value;
}
),
filterSelect(
[
["", "Any ratio"],
@@ -1403,6 +1474,146 @@
});
deadRow.append(deadBox, document.createTextNode(" Hide 4chan links (usually dead)"));
filters.append(deadRow);
// "Do not include": nested collapsibles, so exclusions stay out of the way.
const exclude = document.createElement("details");
exclude.className = "vk-media-exclude";
exclude.append(el("summary", "vk-media-exclude-title", "Do not include"));
const group = title => {
const detailsNode = document.createElement("details");
detailsNode.className = "vk-media-exclude-group";
detailsNode.append(el("summary", "vk-media-status", title));
return detailsNode;
};
const checkRow = (label, checked, onToggle) => {
const row = el("label", "vk-media-check");
const box = document.createElement("input");
box.type = "checkbox";
box.checked = checked;
box.addEventListener("change", () => {
onToggle(box.checked);
renderFilterDrawer();
});
row.append(box, document.createTextNode(` ${label}`));
return row;
};
const toggle = (list, value, on) =>
on ? [...new Set([...list, value])] : list.filter(entry => entry !== value);
const sourceGroup = group("Sources");
for (const [value, label] of [
["twitch", "Twitch"],
["youtube", "YouTube"],
["discord", "Discord"],
["tenor", "Tenor"],
["giphy", "Giphy"]
]) {
sourceGroup.append(
checkRow(label, draft.excludeProviders.includes(value), on => {
draft.excludeProviders = toggle(draft.excludeProviders, value, on);
})
);
}
for (const [value, label] of [
["embed", "Embeds"],
["attachment", "Attachments"],
["sticker", "Stickers"]
]) {
sourceGroup.append(
checkRow(label, draft.excludeSources.includes(value), on => {
draft.excludeSources = toggle(draft.excludeSources, value, on);
})
);
}
exclude.append(sourceGroup);
const extGroup = group("File types");
const extRow = el("div", "vk-media-toolbar");
for (const extension of [
"mp4",
"webm",
"mov",
"mkv",
"png",
"jpg",
"gif",
"webp",
"avif",
"mp3",
"ogg",
"wav",
"m4a",
"flac"
]) {
const chip = el(
"button",
`vk-chip ${draft.excludeExts.includes(extension) ? "vk-chip-off vk-chip-strike" : "vk-chip-on"}`,
extension
);
chip.type = "button";
chip.title = draft.excludeExts.includes(extension)
? `Excluding ${extension}`
: `${extension} included`;
chip.addEventListener("click", () => {
draft.excludeExts = toggle(draft.excludeExts, extension, !draft.excludeExts.includes(extension));
renderFilterDrawer();
});
extRow.append(chip);
}
extGroup.append(extRow);
exclude.append(extGroup);
const amountRow = (label, key, unitKey) => {
const row = el("div", "vk-media-toolbar");
row.append(el("span", "vk-media-status", label));
const input = document.createElement("input");
input.type = "number";
input.min = "0";
input.step = "any";
input.className = "vk-media-input vk-media-small-input";
input.value = draft[key];
input.addEventListener("input", () => {
draft[key] = input.value;
});
row.append(input);
if (unitKey) {
const unit = document.createElement("select");
unit.className = "vk-chip vk-chip-off";
for (const value of Object.keys(SIZE_UNITS)) {
const option = document.createElement("option");
option.value = value;
option.textContent = value;
unit.append(option);
}
unit.value = draft[unitKey];
unit.addEventListener("change", () => {
draft[unitKey] = unit.value;
});
row.append(unit);
} else {
row.append(el("span", "vk-media-status", "seconds"));
}
return row;
};
const sizeGroup = group("Size");
sizeGroup.append(
amountRow("Less than", "sizeLt", "sizeLtUnit"),
amountRow("Greater than", "sizeGt", "sizeGtUnit")
);
exclude.append(sizeGroup);
const lengthGroup = group("Length");
lengthGroup.append(amountRow("Less than", "lenLt"), amountRow("Greater than", "lenGt"));
exclude.append(lengthGroup);
filters.append(exclude);
filterDrawer.append(filters);
// Zoom applies live; it is not part of the filter draft.
@@ -1480,7 +1691,9 @@
function render() {
title.textContent = state.view === "bot" ? "Slopbot" : "Media archive";
toolbar.hidden = false;
// The filter bar and its drawer belong to the Media tab only.
toolbar.hidden = state.view === "bot";
if (state.view === "bot") filterDrawer.hidden = true;
grid.hidden = state.view === "bot";
botPanel.hidden = state.view !== "bot";
scrapeList.hidden = state.view === "bot";
poc/src/control-bridge.ts
+33
View File
@@ -887,6 +887,23 @@ function buildServer(): McpServer {
cachedOnly: z.boolean().optional().describe("Only items with a cached preview"),
exts: z.array(z.string()).optional().describe("Only these file extensions (mp4, webm, png, ...)"),
excludeHosts: z.array(z.string()).optional().describe("Hide URLs from these hosts"),
excludeExts: z.array(z.string()).optional().describe("Hide these file extensions"),
excludeProviders: z
.array(z.string())
.optional()
.describe("Hide these providers (youtube, twitch, discord, tenor, giphy)"),
excludeSources: z
.array(z.string())
.optional()
.describe("Hide these row sources (attachment, sticker, embed)"),
sizeLessThan: z.number().optional().describe("Only items smaller than this many bytes"),
sizeGreaterThan: z.number().optional().describe("Only items larger than this many bytes"),
lengthLessThan: z.number().optional().describe("Only items shorter than this many seconds"),
lengthGreaterThan: z.number().optional().describe("Only items longer than this many seconds"),
sort: z
.enum(["newest", "oldest", "user", "size", "length"])
.optional()
.describe("Order: newest (default), oldest, user, size, length"),
aspect: z
.enum(["vertical", "square", "wide43", "wide169", "ultrawide", "other"])
.optional()
@@ -911,6 +928,14 @@ function buildServer(): McpServer {
cachedOnly,
exts,
excludeHosts,
excludeExts,
excludeProviders,
excludeSources,
sizeLessThan,
sizeGreaterThan,
lengthLessThan,
lengthGreaterThan,
sort,
aspect,
resolution,
after,
@@ -929,6 +954,14 @@ function buildServer(): McpServer {
cachedOnly,
exts,
excludeHosts,
excludeExts,
excludeProviders,
excludeSources,
sizeLessThan,
sizeGreaterThan,
lengthLessThan,
lengthGreaterThan,
sort,
aspect,
resolution,
after,
poc/src/media.ts
+34 -1
View File
@@ -10,6 +10,7 @@ import { isAbsolute, join } from "node:path";
import {
BackfillQueue,
betterCandidate,
sortCandidates,
dirSizeBytes,
isNearLimit,
planRetention,
@@ -22,6 +23,7 @@ import {
type MediaRow,
type BackfillSettings,
type BackfillStatus,
type MediaSort,
type ScrapeBackend,
type ScrapeMeta
} from "../../media-scrape/src/index.js";
@@ -708,6 +710,14 @@ export class MediaService {
cachedOnly?: boolean;
exts?: string[];
excludeHosts?: string[];
excludeExts?: string[];
excludeProviders?: string[];
excludeSources?: string[];
sizeLessThan?: number;
sizeGreaterThan?: number;
lengthLessThan?: number;
lengthGreaterThan?: number;
sort?: MediaSort;
aspect?: string;
resolution?: string;
after?: string;
@@ -829,10 +839,18 @@ export class MediaService {
cachedOnly?: boolean;
exts?: string[];
excludeHosts?: string[];
excludeExts?: string[];
excludeProviders?: string[];
excludeSources?: string[];
sizeLessThan?: number;
sizeGreaterThan?: number;
lengthLessThan?: number;
lengthGreaterThan?: number;
aspect?: string;
resolution?: string;
after?: string;
before?: string;
sort?: MediaSort;
}): Array<{ meta: ScrapeMeta; row: MediaRow }> {
const candidates = new Map<string, { meta: ScrapeMeta; row: MediaRow }>();
@@ -856,6 +874,13 @@ export class MediaService {
cachedOnly: options.cachedOnly,
exts: options.exts,
excludeHosts: options.excludeHosts,
excludeExts: options.excludeExts,
excludeProviders: options.excludeProviders,
excludeSources: options.excludeSources,
sizeLessThan: options.sizeLessThan,
sizeGreaterThan: options.sizeGreaterThan,
lengthLessThan: options.lengthLessThan,
lengthGreaterThan: options.lengthGreaterThan,
aspect: options.aspect,
resolution: options.resolution,
after: options.after,
@@ -893,7 +918,7 @@ export class MediaService {
}
}
return [...candidates.values()];
return sortCandidates([...candidates.values()], options.sort ?? "newest");
}
/** Read media rows across every scrape, richest per URL, newest scrape first. */
@@ -909,6 +934,14 @@ export class MediaService {
cachedOnly?: boolean;
exts?: string[];
excludeHosts?: string[];
excludeExts?: string[];
excludeProviders?: string[];
excludeSources?: string[];
sizeLessThan?: number;
sizeGreaterThan?: number;
lengthLessThan?: number;
lengthGreaterThan?: number;
sort?: MediaSort;
aspect?: string;
resolution?: string;
after?: string;
poc/src/webui.ts
+51 -83
View File
@@ -177,6 +177,54 @@ function startAudioSocket(server: http.Server | https.Server, deps: WebUiDeps):
});
}
/** Shared parsing for the media grid filters, used by /api/media and /api/media/stats. */
function parseMediaFilters(query: Record<string, unknown>) {
const list = (value: unknown) =>
value
? String(value)
.split(",")
.map(entry => entry.trim().toLowerCase())
.filter(Boolean)
: undefined;
const amount = (value: unknown) => {
if (value === undefined || value === "") return undefined;
const number = Number(value);
return Number.isFinite(number) && number >= 0 ? number : undefined;
};
return {
kind: ["image", "gif", "video", "audio"].includes(String(query.kind ?? ""))
? (String(query.kind) as "image" | "gif" | "video" | "audio")
: undefined,
source: ["attachment", "sticker", "embed"].includes(String(query.source ?? ""))
? String(query.source)
: undefined,
channelId: query.channelId ? String(query.channelId) : undefined,
authorIds: list(query.authorId),
origin:
query.origin === "dm" || query.origin === "guild"
? (String(query.origin) as "dm" | "guild")
: undefined,
cachedOnly: query.cachedOnly === "1" || query.cachedOnly === "true",
exts: list(query.ext),
excludeHosts: list(query.excludeHosts),
excludeExts: list(query.excludeExt),
excludeProviders: list(query.excludeProviders),
excludeSources: list(query.excludeSources),
sizeLessThan: amount(query.sizeLt),
sizeGreaterThan: amount(query.sizeGt),
lengthLessThan: amount(query.lenLt),
lengthGreaterThan: amount(query.lenGt),
aspect: query.aspect ? String(query.aspect) : undefined,
resolution: query.resolution ? String(query.resolution) : undefined,
after: query.after ? String(query.after) : undefined,
before: query.before ? String(query.before) : undefined,
sort: ["newest", "oldest", "user", "size", "length"].includes(String(query.sort ?? ""))
? (String(query.sort) as "newest" | "oldest" | "user" | "size" | "length")
: undefined
};
}
export function startWebUi(deps: WebUiDeps): WebUiHandle {
const app = express();
app.use(express.json({ limit: "64kb" }));
@@ -901,58 +949,14 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
app.get(
"/api/media",
wrap(async (req, res) => {
const kind = ["image", "gif", "video", "audio"].includes(String(req.query.kind ?? ""))
? (String(req.query.kind) as "image" | "gif" | "video" | "audio")
: undefined;
const source = ["attachment", "sticker", "embed"].includes(String(req.query.source ?? ""))
? String(req.query.source)
: undefined;
const channelId = req.query.channelId ? String(req.query.channelId) : undefined;
const authorIds = req.query.authorId
? String(req.query.authorId)
.split(",")
.map(value => value.trim())
.filter(Boolean)
: undefined;
const origin =
req.query.origin === "dm" || req.query.origin === "guild"
? (String(req.query.origin) as "dm" | "guild")
: undefined;
const cachedOnly = req.query.cachedOnly === "1" || req.query.cachedOnly === "true";
const exts = req.query.ext
? String(req.query.ext)
.split(",")
.map(value => value.trim().toLowerCase())
.filter(Boolean)
: undefined;
const excludeHosts = req.query.excludeHosts
? String(req.query.excludeHosts)
.split(",")
.map(value => value.trim().toLowerCase())
.filter(Boolean)
: undefined;
const aspect = req.query.aspect ? String(req.query.aspect) : undefined;
const resolution = req.query.resolution ? String(req.query.resolution) : undefined;
const after = req.query.after ? String(req.query.after) : undefined;
const before = req.query.before ? String(req.query.before) : undefined;
const filters = parseMediaFilters(req.query);
const limit = clampLimit(req.query.limit, 60, 500);
const offset = Math.max(0, Number(req.query.offset ?? 0) || 0);
res.json({
ok: true,
result: deps.media.queryMedia({
kind,
source,
channelId,
authorIds,
origin,
cachedOnly,
exts,
excludeHosts,
aspect,
resolution,
after,
before,
...filters,
limit,
offset
})
@@ -970,46 +974,10 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
app.get(
"/api/media/stats",
wrap(async (req, res) => {
const kind = ["image", "gif", "video", "audio"].includes(String(req.query.kind ?? ""))
? (String(req.query.kind) as "image" | "gif" | "video" | "audio")
: undefined;
const authorIds = req.query.authorId
? String(req.query.authorId)
.split(",")
.map(value => value.trim())
.filter(Boolean)
: undefined;
const origin =
req.query.origin === "dm" || req.query.origin === "guild"
? (String(req.query.origin) as "dm" | "guild")
: undefined;
const exts = req.query.ext
? String(req.query.ext)
.split(",")
.map(value => value.trim().toLowerCase())
.filter(Boolean)
: undefined;
const excludeHosts = req.query.excludeHosts
? String(req.query.excludeHosts)
.split(",")
.map(value => value.trim().toLowerCase())
.filter(Boolean)
: undefined;
res.json({
ok: true,
result: deps.media.stats({
kind,
source: req.query.source ? String(req.query.source) : undefined,
channelId: req.query.channelId ? String(req.query.channelId) : undefined,
authorIds,
origin,
cachedOnly: req.query.cachedOnly === "1" || req.query.cachedOnly === "true",
exts,
excludeHosts,
aspect: req.query.aspect ? String(req.query.aspect) : undefined,
resolution: req.query.resolution ? String(req.query.resolution) : undefined,
...parseMediaFilters(req.query),
after: req.query.after ? String(req.query.after) : undefined,
before: req.query.before ? String(req.query.before) : undefined
})
poc/test/media-ui-smoke.mjs
+3
View File
@@ -161,6 +161,9 @@ try {
{ timeout: 20000 }
);
check("media toolbar hidden on the bot tab", !(await page.locator("#media-toolbar").isVisible()));
check("media filter drawer hidden on the bot tab", !(await page.locator(".vk-media-drawer").isVisible()));
const summaryLines = await page.locator(".vk-bot-summary-line").count();
step("backfill summary lines", summaryLines);
check("backfill summary renders", summaryLines >= 3, summaryLines);