Add media storage limits, retention, and the near-limit flow
This commit is contained in:
+2
-1
@@ -30,7 +30,8 @@ mobile draft at 390px.
|
||||
- [x] C4 slopbot backend, tests
|
||||
- [x] C5 daemon wiring: `scrape media` command, MediaService, `/api/media/*`
|
||||
- [x] C6 Media and Bot tabs in the Web UI
|
||||
- [ ] C7 storage limit, warning flow, clean modal, settings
|
||||
- [x] C7a storage limits, retention, warning/confirm flow, storage and clean API
|
||||
- [ ] C7b Settings group: limit field with info hover, usage bar, clean modal
|
||||
- [ ] C8 MCP service hooks
|
||||
- [ ] C9 live verification, tuning, docs
|
||||
- Flextalk voice profile, end to end. The profile window is a movable panel that recenters on each
|
||||
|
||||
@@ -58,6 +58,9 @@ Ordered backlog. `CURRENT.md` tracks the live session state.
|
||||
10. Mobile web UI: first draft at 390px with the four tabs (Users, Chat, Call, Activity) is
|
||||
implemented and rendered headless. Next: move the transcript into the Call tab, put the TTS
|
||||
box in the Call tab as in 1c, and verify with real data on a phone-width viewport.
|
||||
11. TTS VOX preference: a `tts.vox` toggle that exports `VESKSLOP_TTS_VOX` (like `words`), an
|
||||
optional `VOX_CLEAN`, and a help line in the TTS panel. Requested in `TTS-DESIGN.md`; the
|
||||
wrapper side is ready and verified. Inline `/rate` and `/pitch` need no UI.
|
||||
|
||||
## Web UI follow-ups
|
||||
|
||||
|
||||
+8
-3
@@ -1,8 +1,8 @@
|
||||
# Media archive
|
||||
|
||||
Status: design approved 2026-09-16. The engine, the slopbot backend, the `scrape media` command,
|
||||
and the `/api/media/*` routes are implemented (C3-C5). The Media and Bot tabs are C6. Checkpoints
|
||||
live in [CURRENT.md](../CURRENT.md).
|
||||
the `/api/media/*` routes, and storage limits with retention are implemented (C3-C7a). The Media
|
||||
and Bot tabs are C6; the Settings group is C7b. Checkpoints live in [CURRENT.md](../CURRENT.md).
|
||||
|
||||
The media archive saves the media history of a channel. A trusted user runs `scrape media` in the
|
||||
channel. The daemon walks the history through a scrape backend and indexes every image, GIF,
|
||||
@@ -101,12 +101,17 @@ The storage limit covers everything under the DB root: databases, downloads, and
|
||||
Behavior:
|
||||
|
||||
1. At or above the warn threshold, a new `scrape media` command is refused and the reply shows the
|
||||
usage. Re-running the same command accepts. The confirmation lives for a short window.
|
||||
usage. Re-running the same command within 2 minutes accepts. The API mirror is `confirm: true`
|
||||
on `POST /api/media/scrape` (otherwise `409 MEDIA_LIMIT`).
|
||||
2. At or above 100% the oldest completed scrapes are marked `pending-delete`.
|
||||
3. Above limit + 5% overrun, marked scrapes are deleted oldest first until the usage fits.
|
||||
4. No data is deleted inside the 5% overrun band. The hover info (small `i` beside the limit field)
|
||||
explains this.
|
||||
|
||||
The planning is a pure function (`media-scrape/src/retention.ts`) with tests. Only `done`,
|
||||
`failed`, and `paused` scrapes are ever marked; a running scrape is untouched. Deletion removes the
|
||||
scrape directory and forgets it in the engine.
|
||||
|
||||
**Clean media** in Settings lists exactly what will be deleted, with sizes and counts, and states
|
||||
that deletion is irreversible. The delete button is disabled for a 5 second countdown.
|
||||
|
||||
|
||||
+3
-1
@@ -249,8 +249,10 @@ replaceable.
|
||||
| GET | `/api/media` | `kind?`, `limit?`, `offset?` | Media items across every scrape, newest first |
|
||||
| GET | `/api/media/scrapes` | none | Every scrape with state and counters |
|
||||
| GET | `/api/media/status` | none | Slopbot connection state and the scrape list |
|
||||
| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?` | Start a media scrape |
|
||||
| GET | `/api/media/storage` | none | Usage against the limit, per-kind sizes, `nearLimit`; marks old scrapes when over |
|
||||
| POST | `/api/media/scrape` | `channelId`, `guildId?`, `mode?`, `confirm?` | Start a media scrape. Near the limit: `409` with code `MEDIA_LIMIT` unless `confirm: true` |
|
||||
| POST | `/api/media/scrape/stop` | `id` | Stop a media scrape |
|
||||
| POST | `/api/media/clean` | `ids?` | Delete the given scrapes, or every scrape when `ids` is absent |
|
||||
| GET | `/api/logs` | `limit` | The persistent action log |
|
||||
| GET | `/api/events` | none | Server-sent events |
|
||||
|
||||
|
||||
@@ -18,7 +18,8 @@ import {
|
||||
type BackendName,
|
||||
type CoreMessage,
|
||||
type ScrapeMeta,
|
||||
type ScrapeMode
|
||||
type ScrapeMode,
|
||||
type ScrapeState
|
||||
} from "./types.js";
|
||||
|
||||
export interface FetchPage {
|
||||
@@ -146,6 +147,24 @@ export class ScrapeEngine {
|
||||
this.stopRequested.add(id);
|
||||
}
|
||||
|
||||
/** Persist a state change made outside a job, such as retention marking. */
|
||||
setState(id: string, state: ScrapeState): ScrapeMeta | null {
|
||||
const meta = this.active.get(id) ?? readScrapeMeta(join(this.options.rootDir, id));
|
||||
if (!meta) return null;
|
||||
|
||||
meta.state = state;
|
||||
meta.updatedAt = new Date(this.now()).toISOString();
|
||||
writeScrapeMeta(join(this.options.rootDir, id), meta);
|
||||
this.active.set(id, meta);
|
||||
return { ...meta };
|
||||
}
|
||||
|
||||
/** Drop a deleted scrape from the in-memory list. */
|
||||
forget(id: string): void {
|
||||
this.active.delete(id);
|
||||
this.stopRequested.delete(id);
|
||||
}
|
||||
|
||||
private async runJob(job: QueuedJob): Promise<void> {
|
||||
const { meta, dir, backend } = job;
|
||||
const store = new ScrapeStore(dir);
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
|
||||
export { extractMedia, type ExtractedMedia } from "./extract.js";
|
||||
export { isNearLimit, planRetention, type RetentionPlan, type RetentionScrape } from "./retention.js";
|
||||
export {
|
||||
ScrapeEngine,
|
||||
type FetchPage,
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Pure retention planning for the media archive. Rules: docs/media.md.
|
||||
*/
|
||||
|
||||
export interface RetentionScrape {
|
||||
id: string;
|
||||
state: string;
|
||||
createdAt: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface RetentionPlan {
|
||||
/** Completed scrapes to mark pending-delete so the usage can fit the limit. */
|
||||
mark: string[];
|
||||
/** Marked scrapes to delete now: the usage is past the overrun allowance. */
|
||||
delete: string[];
|
||||
totalBytes: number;
|
||||
}
|
||||
|
||||
/** States that may be marked for deletion. A running scrape is never touched. */
|
||||
const MARKABLE = new Set(["done", "failed", "paused"]);
|
||||
|
||||
export function isNearLimit(usedBytes: number, limitBytes: number, warnPercent: number): boolean {
|
||||
return usedBytes >= limitBytes * (warnPercent / 100);
|
||||
}
|
||||
|
||||
export function planRetention(
|
||||
scrapes: RetentionScrape[],
|
||||
limitBytes: number,
|
||||
overrunPercent: number
|
||||
): RetentionPlan {
|
||||
const totalBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
|
||||
const oldest = [...scrapes].sort((a, b) => (a.createdAt < b.createdAt ? -1 : 1));
|
||||
const overrunLimit = limitBytes * (1 + overrunPercent / 100);
|
||||
|
||||
// Past the allowance: delete marked scrapes, oldest first, until the usage fits.
|
||||
if (totalBytes > overrunLimit) {
|
||||
const remove: string[] = [];
|
||||
let planned = totalBytes;
|
||||
|
||||
for (const scrape of oldest) {
|
||||
if (planned <= limitBytes) break;
|
||||
if (scrape.state !== "pending-delete") continue;
|
||||
remove.push(scrape.id);
|
||||
planned -= scrape.bytes;
|
||||
}
|
||||
|
||||
return { mark: [], delete: remove, totalBytes };
|
||||
}
|
||||
|
||||
if (totalBytes <= limitBytes) return { mark: [], delete: [], totalBytes };
|
||||
|
||||
// Between the limit and the allowance: mark oldest completed scrapes, no deletion.
|
||||
const mark: string[] = [];
|
||||
let planned = totalBytes;
|
||||
|
||||
for (const scrape of oldest) {
|
||||
if (planned <= limitBytes) break;
|
||||
if (!MARKABLE.has(scrape.state)) continue;
|
||||
mark.push(scrape.id);
|
||||
planned -= scrape.bytes;
|
||||
}
|
||||
|
||||
return { mark, delete: [], totalBytes };
|
||||
}
|
||||
@@ -216,6 +216,19 @@ export class ScrapeStore {
|
||||
return added;
|
||||
}
|
||||
|
||||
/** Sum of the recorded source sizes per kind. Used by the storage bar. */
|
||||
sumBytesByKind(): Record<MediaKind, number> {
|
||||
const rows = this.db
|
||||
.prepare("SELECT kind, COALESCE(SUM(size), 0) AS bytes FROM media GROUP BY kind")
|
||||
.all() as unknown as Array<{ kind: string; bytes: number }>;
|
||||
|
||||
const result = emptyMediaCounts();
|
||||
for (const row of rows) {
|
||||
if (row.kind in result) result[row.kind as MediaKind] = row.bytes;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
counts(): Record<MediaKind, number> {
|
||||
const rows = this.db
|
||||
.prepare("SELECT kind, COUNT(*) AS n FROM media GROUP BY kind")
|
||||
|
||||
@@ -10,6 +10,7 @@ import { join } from "node:path";
|
||||
import { ScrapeEngine, type FetchPage, type ScrapeBackend } from "../src/engine.js";
|
||||
import { extractMedia } from "../src/extract.js";
|
||||
import { PuppetBackend } from "../src/puppet-backend.js";
|
||||
import { isNearLimit, planRetention } from "../src/retention.js";
|
||||
import { dirSizeBytes, listScrapes, ScrapeStore } from "../src/store.js";
|
||||
import type { CoreAttachment, CoreMessage } from "../src/types.js";
|
||||
|
||||
@@ -173,7 +174,9 @@ void (async () => {
|
||||
const backend = new FakeBackend([
|
||||
{
|
||||
messages: [
|
||||
message("100", { attachments: [attachment({ filename: "one.png", contentType: "image/png" })] }),
|
||||
message("100", {
|
||||
attachments: [attachment({ filename: "one.png", contentType: "image/png", size: 10 })]
|
||||
}),
|
||||
message("101")
|
||||
]
|
||||
},
|
||||
@@ -202,6 +205,7 @@ void (async () => {
|
||||
assert.equal(stored.counts().image, 1);
|
||||
assert.equal(stored.queryMedia({ kind: "image" }).length, 1);
|
||||
assert.equal(stored.queryMedia({ kind: "video" }).length, 0);
|
||||
assert.equal(stored.sumBytesByKind().image, 10);
|
||||
stored.close();
|
||||
|
||||
assert.equal(listScrapes(root).length, 1);
|
||||
@@ -272,6 +276,42 @@ void (async () => {
|
||||
assert.deepEqual(calls[0], { channelId: "123", limit: 5, before: "60" });
|
||||
});
|
||||
|
||||
await test("retention marks the oldest completed scrapes up to the limit", () => {
|
||||
const scrapes = [
|
||||
{ id: "old", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
|
||||
{ id: "new", state: "done", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
|
||||
{ id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
|
||||
];
|
||||
|
||||
const plan = planRetention(scrapes, 1000, 100);
|
||||
assert.deepEqual(plan.mark, ["old", "new"]);
|
||||
assert.deepEqual(plan.delete, []);
|
||||
});
|
||||
|
||||
await test("retention deletes marked scrapes past the overrun allowance", () => {
|
||||
const scrapes = [
|
||||
{ id: "old", state: "pending-delete", createdAt: "2026-09-01T00:00:00.000Z", bytes: 600 },
|
||||
{ id: "new", state: "pending-delete", createdAt: "2026-09-02T00:00:00.000Z", bytes: 600 },
|
||||
{ id: "run", state: "running", createdAt: "2026-09-03T00:00:00.000Z", bytes: 600 }
|
||||
];
|
||||
|
||||
const plan = planRetention(scrapes, 1000, 5);
|
||||
assert.deepEqual(plan.delete, ["old", "new"]);
|
||||
assert.deepEqual(plan.mark, []);
|
||||
});
|
||||
|
||||
await test("retention leaves usage under the limit alone", () => {
|
||||
const plan = planRetention(
|
||||
[{ id: "one", state: "done", createdAt: "2026-09-01T00:00:00.000Z", bytes: 500 }],
|
||||
1000,
|
||||
5
|
||||
);
|
||||
|
||||
assert.deepEqual(plan, { mark: [], delete: [], totalBytes: 500 });
|
||||
assert.equal(isNearLimit(900, 1000, 90), true);
|
||||
assert.equal(isNearLimit(899, 1000, 90), false);
|
||||
});
|
||||
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
|
||||
if (failures > 0) {
|
||||
|
||||
@@ -8,5 +8,6 @@ setup/out-*.reg
|
||||
*.wav
|
||||
# Generated per-checkout configuration.
|
||||
setup/microslopsam.env
|
||||
__pycache__/
|
||||
# Engine DLLs and voice data are proprietary; never commit them.
|
||||
third_party/
|
||||
|
||||
@@ -43,6 +43,7 @@ let connecting = false;
|
||||
const media = new MediaService({
|
||||
getCore: () => core,
|
||||
isCoreConnected: () => coreConnected,
|
||||
getMediaPrefs: () => preferences.get().media,
|
||||
log: line => console.log(`[media] ${line}`)
|
||||
});
|
||||
|
||||
|
||||
@@ -5,7 +5,11 @@
|
||||
* (later) MCP tools call.
|
||||
*/
|
||||
|
||||
import { rmSync } from "node:fs";
|
||||
import {
|
||||
dirSizeBytes,
|
||||
isNearLimit,
|
||||
planRetention,
|
||||
ScrapeEngine,
|
||||
ScrapeStore,
|
||||
scrapeDir,
|
||||
@@ -25,12 +29,14 @@ import {
|
||||
type SlopbotStatus
|
||||
} from "../../media-scrape/slopbot/index.js";
|
||||
import type { CoreClient } from "./core-client.js";
|
||||
import type { MediaPreferences } from "./preferences.js";
|
||||
|
||||
type BotClient = Awaited<ReturnType<typeof startSlopbot>>;
|
||||
|
||||
export interface MediaServiceDeps {
|
||||
getCore: () => CoreClient | undefined;
|
||||
isCoreConnected: () => boolean;
|
||||
getMediaPrefs: () => MediaPreferences;
|
||||
log?: (line: string) => void;
|
||||
}
|
||||
|
||||
@@ -39,6 +45,44 @@ export interface StartScrapeInput {
|
||||
guildId?: string | null;
|
||||
mode?: "links" | "download";
|
||||
backend?: "auto" | "slopbot" | "puppet";
|
||||
/** Set by a re-run of the command after the near-limit warning. */
|
||||
confirm?: boolean;
|
||||
}
|
||||
|
||||
export interface ScrapeBytes {
|
||||
id: string;
|
||||
channelId: string;
|
||||
state: string;
|
||||
createdAt: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface MediaStorageStatus {
|
||||
usedBytes: number;
|
||||
limitBytes: number;
|
||||
warnPercent: number;
|
||||
overrunPercent: number;
|
||||
nearLimit: boolean;
|
||||
byKind: Record<MediaKind, number>;
|
||||
scrapes: ScrapeBytes[];
|
||||
}
|
||||
|
||||
/** Thrown when a scrape starts near the storage limit without confirmation. */
|
||||
export class MediaLimitError extends Error {
|
||||
readonly code = "MEDIA_LIMIT";
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
readonly usedBytes: number,
|
||||
readonly limitBytes: number
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
const gib = bytes / 1024 ** 3;
|
||||
return gib >= 1 ? `${gib.toFixed(2)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`;
|
||||
}
|
||||
|
||||
export interface MediaItemResult {
|
||||
@@ -104,7 +148,92 @@ export class MediaService {
|
||||
});
|
||||
}
|
||||
|
||||
/** Combined size of every scrape directory, with the limit settings applied. */
|
||||
storageStatus(): MediaStorageStatus {
|
||||
const prefs = this.deps.getMediaPrefs();
|
||||
const byKind: Record<MediaKind, number> = { image: 0, gif: 0, video: 0, audio: 0 };
|
||||
|
||||
const scrapes = this.engine.list().map(meta => {
|
||||
const dir = scrapeDir(this.rootDir, meta.id);
|
||||
const bytes = dirSizeBytes(dir);
|
||||
|
||||
try {
|
||||
const store = new ScrapeStore(dir);
|
||||
try {
|
||||
const sums = store.sumBytesByKind();
|
||||
for (const kind of Object.keys(byKind) as MediaKind[]) byKind[kind] += sums[kind];
|
||||
} finally {
|
||||
store.close();
|
||||
}
|
||||
} catch {
|
||||
// an unreadable index counts as zero bytes
|
||||
}
|
||||
|
||||
return { id: meta.id, channelId: meta.channelId, state: meta.state, createdAt: meta.createdAt, bytes };
|
||||
});
|
||||
|
||||
const usedBytes = scrapes.reduce((sum, scrape) => sum + scrape.bytes, 0);
|
||||
|
||||
return {
|
||||
usedBytes,
|
||||
limitBytes: prefs.limitBytes,
|
||||
warnPercent: prefs.warnPercent,
|
||||
overrunPercent: prefs.overrunPercent,
|
||||
nearLimit: isNearLimit(usedBytes, prefs.limitBytes, prefs.warnPercent),
|
||||
byKind,
|
||||
scrapes
|
||||
};
|
||||
}
|
||||
|
||||
/** Mark the oldest completed scrapes when the usage is at or over the limit. */
|
||||
refreshRetention(): MediaStorageStatus {
|
||||
const status = this.storageStatus();
|
||||
const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
|
||||
for (const id of plan.mark) this.engine.setState(id, "pending-delete");
|
||||
return this.storageStatus();
|
||||
}
|
||||
|
||||
private applyRetention(): void {
|
||||
const status = this.storageStatus();
|
||||
const plan = planRetention(status.scrapes, status.limitBytes, status.overrunPercent);
|
||||
for (const id of plan.mark) this.engine.setState(id, "pending-delete");
|
||||
for (const id of plan.delete) this.deleteScrape(id);
|
||||
}
|
||||
|
||||
deleteScrape(id: string): boolean {
|
||||
try {
|
||||
rmSync(scrapeDir(this.rootDir, id), { recursive: true, force: true });
|
||||
this.engine.forget(id);
|
||||
this.deps.log?.(`deleted scrape ${id}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.deps.log?.(`delete failed for ${id}: ${String(error)}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Delete the given scrapes, or every scrape when no IDs are given. */
|
||||
cleanMedia(ids?: string[]): number {
|
||||
const targets = ids?.length ? ids : this.engine.list().map(meta => meta.id);
|
||||
let deleted = 0;
|
||||
for (const id of targets) {
|
||||
if (this.deleteScrape(id)) deleted++;
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
|
||||
async startScrape(input: StartScrapeInput): Promise<ScrapeMeta> {
|
||||
this.applyRetention();
|
||||
|
||||
const status = this.storageStatus();
|
||||
if (status.nearLimit && !input.confirm) {
|
||||
throw new MediaLimitError(
|
||||
`media storage is at ${formatBytes(status.usedBytes)} of ${formatBytes(status.limitBytes)}`,
|
||||
status.usedBytes,
|
||||
status.limitBytes
|
||||
);
|
||||
}
|
||||
|
||||
const mode = input.mode ?? "links";
|
||||
let backend: ScrapeBackend;
|
||||
|
||||
|
||||
+42
-1
@@ -76,12 +76,22 @@ export interface AudioPreferences {
|
||||
duck: boolean;
|
||||
}
|
||||
|
||||
export interface MediaPreferences {
|
||||
/** Combined size limit for the media DB root: databases, downloads, previews. */
|
||||
limitBytes: number;
|
||||
/** Warn threshold in percent, 1-100. */
|
||||
warnPercent: number;
|
||||
/** No-delete overrun allowance in percent, 0-100. */
|
||||
overrunPercent: number;
|
||||
}
|
||||
|
||||
export interface WebPreferences {
|
||||
/** null means every guild is visible. An empty array hides all. */
|
||||
visibleGuilds: string[] | null;
|
||||
tts: TtsPreferences;
|
||||
audio: AudioPreferences;
|
||||
favourites: FavouritesPreferences;
|
||||
media: MediaPreferences;
|
||||
}
|
||||
|
||||
const DEFAULTS: WebPreferences = {
|
||||
@@ -101,9 +111,35 @@ const DEFAULTS: WebPreferences = {
|
||||
wordGapMs: null
|
||||
},
|
||||
audio: { duck: true },
|
||||
favourites: { users: [], channels: [] }
|
||||
favourites: { users: [], channels: [] },
|
||||
media: {
|
||||
limitBytes: 30 * 1024 * 1024 * 1024,
|
||||
warnPercent: 90,
|
||||
overrunPercent: 5
|
||||
}
|
||||
};
|
||||
|
||||
const MEDIA_LIMIT_MIN = 256 * 1024 * 1024;
|
||||
const MEDIA_LIMIT_MAX = 10 * 1024 * 1024 * 1024 * 1024;
|
||||
|
||||
function normalizeMedia(raw: any, fallback: MediaPreferences): MediaPreferences {
|
||||
const limit = Number(raw?.limitBytes);
|
||||
const warn = Number(raw?.warnPercent);
|
||||
const overrun = Number(raw?.overrunPercent);
|
||||
|
||||
return {
|
||||
limitBytes:
|
||||
Number.isFinite(limit) && limit >= MEDIA_LIMIT_MIN && limit <= MEDIA_LIMIT_MAX
|
||||
? Math.round(limit)
|
||||
: fallback.limitBytes,
|
||||
warnPercent: Number.isFinite(warn) && warn >= 1 && warn <= 100 ? Math.round(warn) : fallback.warnPercent,
|
||||
overrunPercent:
|
||||
Number.isFinite(overrun) && overrun >= 0 && overrun <= 100
|
||||
? Math.round(overrun)
|
||||
: fallback.overrunPercent
|
||||
};
|
||||
}
|
||||
|
||||
export interface PreferencesOptions {
|
||||
/** The old state file that may still hold preferences. */
|
||||
legacyFile?: string;
|
||||
@@ -199,6 +235,10 @@ export class PreferencesStore {
|
||||
};
|
||||
}
|
||||
|
||||
if (patch.media !== undefined) {
|
||||
this.preferences.media = normalizeMedia(patch.media, this.preferences.media);
|
||||
}
|
||||
|
||||
if (patch.favourites !== undefined) {
|
||||
this.preferences.favourites = {
|
||||
users: Array.isArray(patch.favourites?.users)
|
||||
@@ -255,6 +295,7 @@ export class PreferencesStore {
|
||||
audio: {
|
||||
duck: raw?.audio?.duck !== false
|
||||
},
|
||||
media: normalizeMedia(raw?.media, DEFAULTS.media),
|
||||
favourites: {
|
||||
users: Array.isArray(raw?.favourites?.users)
|
||||
? [...new Set<string>(raw.favourites.users.map((id: unknown) => String(id)))].slice(0, 100)
|
||||
|
||||
+19
-1
@@ -123,6 +123,7 @@ export interface PuppetMedia {
|
||||
channelId: string;
|
||||
guildId: string | null;
|
||||
mode: "links" | "download";
|
||||
confirm?: boolean;
|
||||
}): Promise<{ id: string }>;
|
||||
stopScrape(id: string): void;
|
||||
listScrapes(): PuppetScrape[];
|
||||
@@ -174,6 +175,8 @@ export class Puppet {
|
||||
private readonly lastFetchAt = new Map<string, number>();
|
||||
private readonly pending = new Map<string, { entryId: string; expiresAt: number }>();
|
||||
private readonly polls = new Map<string, PollState>();
|
||||
/** Near-limit scrape warnings waiting for the same user to re-run the command. */
|
||||
private readonly scrapeConfirm = new Map<string, number>();
|
||||
|
||||
private voiceChannelId?: string | null;
|
||||
private ownerVoiceChannelId?: string | null;
|
||||
@@ -753,11 +756,26 @@ export class Puppet {
|
||||
return this.respond(authorId, channelId, "scrape", "Download mode needs the admin tier.");
|
||||
}
|
||||
|
||||
const key = `${channelId}:${authorId}`;
|
||||
const confirmUntil = this.scrapeConfirm.get(key);
|
||||
const confirm = confirmUntil !== undefined && Date.now() < confirmUntil;
|
||||
if (confirm) this.scrapeConfirm.delete(key);
|
||||
|
||||
try {
|
||||
const scrape = await this.media.startScrape({ channelId, guildId: guildId ?? null, mode });
|
||||
const scrape = await this.media.startScrape({ channelId, guildId: guildId ?? null, mode, confirm });
|
||||
this.actionLog.append({ type: "media", action: "scrape", channelId, mode, scrapeId: scrape.id });
|
||||
return this.respond(authorId, channelId, "scrape", `Media scrape started (${mode}): ${scrape.id}.`);
|
||||
} catch (error) {
|
||||
if ((error as { code?: string }).code === "MEDIA_LIMIT" && !confirm) {
|
||||
this.scrapeConfirm.set(key, Date.now() + 120_000);
|
||||
return this.respond(
|
||||
authorId,
|
||||
channelId,
|
||||
"scrape",
|
||||
`${(error as Error).message}. Run the command again within 2 minutes to accept.`
|
||||
);
|
||||
}
|
||||
|
||||
this.actionLog.append({ type: "error", scope: "media", channelId, message: String(error) });
|
||||
return this.respond(
|
||||
authorId,
|
||||
|
||||
+46
-3
@@ -20,7 +20,7 @@ import type { AuthService, AuthStatus } from "./auth-service.js";
|
||||
import type { AudioService } from "./audio-service.js";
|
||||
import { CoreError, type CoreClient } from "./core-client.js";
|
||||
import type { CoreProcess } from "./core-process.js";
|
||||
import type { MediaService } from "./media.js";
|
||||
import { MediaLimitError, type MediaService } from "./media.js";
|
||||
import { dedupe, getQueryCount, paced, pacedSend, pacedVoice } from "./pacing.js";
|
||||
import type { PreferencesStore, WebPreferences } from "./preferences.js";
|
||||
import type { Puppet } from "./puppet.js";
|
||||
@@ -206,6 +206,16 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof MediaLimitError) {
|
||||
res.status(409).json({
|
||||
ok: false,
|
||||
code: error.code,
|
||||
error: error.message,
|
||||
result: { usedBytes: error.usedBytes, limitBytes: error.limitBytes }
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const status = error instanceof HttpError ? error.status : 500;
|
||||
res.status(status).json({ ok: false, error: String(error?.message ?? error) });
|
||||
});
|
||||
@@ -403,7 +413,7 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
app.post(
|
||||
"/api/preferences",
|
||||
wrap(async (req, res) => {
|
||||
const { visibleGuilds, tts, audio, favourites } = req.body ?? {};
|
||||
const { visibleGuilds, tts, audio, favourites, media: mediaPrefs } = req.body ?? {};
|
||||
|
||||
if (visibleGuilds !== undefined && visibleGuilds !== null && !Array.isArray(visibleGuilds)) {
|
||||
throw new HttpError(400, "visibleGuilds must be an array of guild IDs or null");
|
||||
@@ -438,6 +448,21 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
};
|
||||
}
|
||||
|
||||
if (mediaPrefs !== undefined) {
|
||||
const current = deps.preferences.get().media;
|
||||
patch.media = {
|
||||
limitBytes: Number.isFinite(Number(mediaPrefs?.limitBytes))
|
||||
? Number(mediaPrefs.limitBytes)
|
||||
: current.limitBytes,
|
||||
warnPercent: Number.isFinite(Number(mediaPrefs?.warnPercent))
|
||||
? Number(mediaPrefs.warnPercent)
|
||||
: current.warnPercent,
|
||||
overrunPercent: Number.isFinite(Number(mediaPrefs?.overrunPercent))
|
||||
? Number(mediaPrefs.overrunPercent)
|
||||
: current.overrunPercent
|
||||
};
|
||||
}
|
||||
|
||||
if (audio !== undefined) {
|
||||
patch.audio = { duck: audio?.duck !== false };
|
||||
deps.audio.setDuckEnabled(patch.audio.duck);
|
||||
@@ -729,6 +754,23 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/media/storage",
|
||||
wrap(async (_req, res) => {
|
||||
res.json({ ok: true, result: deps.media.refreshRetention() });
|
||||
})
|
||||
);
|
||||
|
||||
app.post(
|
||||
"/api/media/clean",
|
||||
wrap(async (req, res) => {
|
||||
const ids = Array.isArray(req.body?.ids) ? req.body.ids.map(String) : undefined;
|
||||
const deleted = deps.media.cleanMedia(ids);
|
||||
deps.getPuppet()?.appendAction({ type: "media", action: "clean", deleted });
|
||||
res.json({ ok: true, result: { deleted } });
|
||||
})
|
||||
);
|
||||
|
||||
app.get(
|
||||
"/api/media",
|
||||
wrap(async (req, res) => {
|
||||
@@ -750,8 +792,9 @@ export function startWebUi(deps: WebUiDeps): WebUiHandle {
|
||||
|
||||
const mode = req.body?.mode === "download" ? "download" : "links";
|
||||
const guildId = req.body?.guildId ? String(req.body.guildId) : null;
|
||||
const confirm = req.body?.confirm === true;
|
||||
|
||||
const scrape = await deps.media.startScrape({ channelId, guildId, mode });
|
||||
const scrape = await deps.media.startScrape({ channelId, guildId, mode, confirm });
|
||||
deps.getPuppet()?.appendAction({
|
||||
type: "media",
|
||||
action: "scrape",
|
||||
|
||||
Reference in New Issue
Block a user