Cache embed thumbnails in download mode
This commit is contained in:
Fisto
+76
-11
@@ -351,7 +351,7 @@ void (async () => {
|
||||
provider: "YouTube",
|
||||
videoUrl: "https://www.youtube.com/embed/x",
|
||||
imageUrl: null,
|
||||
thumbnailUrl: null
|
||||
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
}
|
||||
]
|
||||
})
|
||||
@@ -359,6 +359,7 @@ void (async () => {
|
||||
}
|
||||
]);
|
||||
|
||||
const fetched: string[] = [];
|
||||
const engine = new ScrapeEngine({
|
||||
rootDir: root,
|
||||
pageSize: 2,
|
||||
@@ -367,10 +368,15 @@ void (async () => {
|
||||
sleep: async () => {},
|
||||
random: () => 0,
|
||||
log: () => {},
|
||||
previews: false,
|
||||
previews: true,
|
||||
download: {
|
||||
fetchImpl: async () =>
|
||||
new Response("png-bytes", { status: 200, headers: { "content-type": "image/png" } }),
|
||||
fetchImpl: async (input: any) => {
|
||||
fetched.push(String(input));
|
||||
return new Response("png-bytes", {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" }
|
||||
});
|
||||
},
|
||||
minGapMs: 0,
|
||||
maxGapMs: 0,
|
||||
sleep: async () => {}
|
||||
@@ -383,6 +389,9 @@ void (async () => {
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "done");
|
||||
assert.equal(meta?.downloaded, 1);
|
||||
assert.equal(meta?.previewed, 2);
|
||||
assert.equal(fetched.some(url => url.includes("youtube.com/watch")), false);
|
||||
assert.equal(fetched.some(url => url.includes("i.ytimg.com")), true);
|
||||
|
||||
const store = new ScrapeStore(join(root, started.id));
|
||||
const row = store.queryMedia({ kind: "image" })[0];
|
||||
@@ -426,13 +435,20 @@ void (async () => {
|
||||
store.close();
|
||||
});
|
||||
|
||||
await test("preview urls are Discord-only resizes", () => {
|
||||
await test("preview urls resize Discord and pass embed thumbnails through", () => {
|
||||
assert.equal(
|
||||
previewUrlFor({ url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" }),
|
||||
"https://media.discordapp.net/attachments/1/2/pic.png?width=320&height=320&format=webp"
|
||||
);
|
||||
assert.equal(previewUrlFor({ url: "https://www.youtube.com/watch?v=x", kind: "video" }), null);
|
||||
assert.equal(previewUrlFor({ url: "https://i.ytimg.com/vi/x/hq.jpg", kind: "image" }), null);
|
||||
assert.equal(
|
||||
previewUrlFor({
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
kind: "video",
|
||||
previewUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
}),
|
||||
"https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
);
|
||||
});
|
||||
|
||||
await test("preview downloads are written by hash", async () => {
|
||||
@@ -440,7 +456,13 @@ void (async () => {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
const outcome = await downloadPreview(
|
||||
{ id: 9, url: "https://cdn.discordapp.com/attachments/1/2/pic.png", kind: "image" },
|
||||
{
|
||||
id: 9,
|
||||
url: "https://cdn.discordapp.com/attachments/1/2/pic.png",
|
||||
kind: "image",
|
||||
source: "attachment",
|
||||
preview_url: null
|
||||
},
|
||||
dir,
|
||||
{
|
||||
fetchImpl: async () =>
|
||||
@@ -453,17 +475,57 @@ void (async () => {
|
||||
assert.equal(existsSync(join(dir, `${outcome.hash}.webp`)), true);
|
||||
});
|
||||
|
||||
await test("engine caches previews in links mode", async () => {
|
||||
await test("embed thumbnails download only when enabled", async () => {
|
||||
const dir = join(root, "preview-embed");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
const media = {
|
||||
id: 11,
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
kind: "video" as const,
|
||||
source: "embed",
|
||||
preview_url: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
};
|
||||
const calls: string[] = [];
|
||||
const fetchImpl = async (input: any) => {
|
||||
calls.push(String(input));
|
||||
return new Response("thumb", { status: 200, headers: { "content-type": "image/webp" } });
|
||||
};
|
||||
|
||||
const off = await downloadPreview(media, dir, { fetchImpl });
|
||||
assert.equal(off.state, "skipped");
|
||||
assert.equal(calls.length, 0);
|
||||
|
||||
const on = await downloadPreview(media, dir, { fetchImpl, embedPreviews: true });
|
||||
assert.equal(on.state, "downloaded");
|
||||
assert.equal(calls[0], "https://i.ytimg.com/vi/x/hqdefault.jpg");
|
||||
assert.equal(calls.some(url => url.includes("youtube.com/watch")), false);
|
||||
});
|
||||
|
||||
await test("engine caches previews in links mode and skips embeds", async () => {
|
||||
const backend = new FakeBackend([
|
||||
{
|
||||
messages: [
|
||||
message("600", {
|
||||
attachments: [attachment({ filename: "prev.png", contentType: "image/png" })]
|
||||
attachments: [attachment({ filename: "prev.png", contentType: "image/png" })],
|
||||
embeds: [
|
||||
{
|
||||
type: "video",
|
||||
url: "https://www.youtube.com/watch?v=x",
|
||||
title: null,
|
||||
provider: "YouTube",
|
||||
videoUrl: "https://www.youtube.com/embed/x",
|
||||
imageUrl: null,
|
||||
thumbnailUrl: "https://i.ytimg.com/vi/x/hqdefault.jpg"
|
||||
}
|
||||
]
|
||||
})
|
||||
]
|
||||
}
|
||||
]);
|
||||
|
||||
const fetched: string[] = [];
|
||||
|
||||
const engine = new ScrapeEngine({
|
||||
rootDir: root,
|
||||
pageSize: 2,
|
||||
@@ -474,8 +536,10 @@ void (async () => {
|
||||
log: () => {},
|
||||
previews: true,
|
||||
download: {
|
||||
fetchImpl: async () =>
|
||||
new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } }),
|
||||
fetchImpl: async (input: any) => {
|
||||
fetched.push(String(input));
|
||||
return new Response("webp-bytes", { status: 200, headers: { "content-type": "image/webp" } });
|
||||
},
|
||||
minGapMs: 0,
|
||||
maxGapMs: 0,
|
||||
sleep: async () => {}
|
||||
@@ -488,6 +552,7 @@ void (async () => {
|
||||
const meta = engine.get(started.id);
|
||||
assert.equal(meta?.state, "done");
|
||||
assert.equal(meta?.previewed, 1);
|
||||
assert.equal(fetched.some(url => url.includes("ytimg.com")), false);
|
||||
|
||||
const store = new ScrapeStore(join(root, started.id));
|
||||
const row = store.queryMedia({ kind: "image" })[0];
|
||||
|
||||
Reference in New Issue
Block a user