Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Persist in-memory cache, change hash. #116

Merged
merged 11 commits into from
Aug 30, 2021
Merged

Persist in-memory cache, change hash. #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
b299cda
Add FS cache, change hashing
zeroby0 Jul 19, 2021
dfc456f
Update hash in tests
zeroby0 Jul 19, 2021
f265638
Simplify hashing, and update hashes
zeroby0 Jul 19, 2021
4332dec
Support svgShortCircuit
zeroby0 Aug 4, 2021
b0ee0a9
change hash, add public api
zeroby0 Aug 6, 2021
27a1a2b
in-memory memoize hash
zeroby0 Aug 6, 2021
9d13b54
fix hashes in tests
zeroby0 Aug 6, 2021
2f0f27e
Support node v12 base64url
zeroby0 Aug 7, 2021
091f430
Use remote asset content in hash
zeroby0 Aug 7, 2021
2f5f532
Check for options.remoteAssetContent
zeroby0 Aug 11, 2021
20d7905
Merge branch 'master' into cache
zeroby0 Aug 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 70 additions & 3 deletions img.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ const path = require("path");
const fs = require("fs");
const fsp = fs.promises;
const { URL } = require("url");
const shorthash = require("short-hash");
const { createHash } = require("crypto");
const {default: PQueue} = require("p-queue");
const base64url = require("base64url");
const getImageSize = require("image-size");
const sharp = require("sharp");
const debug = require("debug")("EleventyImg");
Expand Down Expand Up @@ -123,8 +124,52 @@ function getValidWidths(originalWidth, widths = [], allowUpscale = false) {
return filtered.sort((a, b) => a - b);
}

let imgHashCache = {};
function getHash(src, imgOptions={}, length=10) {
if(src in imgHashCache) return imgHashCache[src];

if(typeof src === "string" && isFullUrl(src) && !("remoteAssetContent" in imgOptions)) {
throw new Error("When using getHash with URLs, imgOptions.remoteAssetContent should be set to the content of the remote asset.");
}

const hash = createHash("sha256");

let opts = Object.assign({
"userOptions": {},
"sharpOptions": {},
"sharpWebpOptions": {},
"sharpPngOptions": {},
"sharpJpegOptions": {},
"sharpAvifOptions": {},
"remoteAssetContent": {}
}, imgOptions);

opts = {
userOptions: opts.userOptions,
sharpOptions: opts.sharpOptions,
sharpWebpOptions: opts.sharpWebpOptions,
sharpPngOptions: opts.sharpPngOptions,
sharpJpegOptions: opts.sharpJpegOptions,
sharpAvifOptions: opts.sharpAvifOptions,
remoteAssetContent: opts.remoteAssetContent
};

if(fs.existsSync(src)) {
const fileContent = fs.readFileSync(src);
zeroby0 marked this conversation as resolved.
Show resolved Hide resolved
hash.update(fileContent);
} else {
hash.update(src);
Copy link
Contributor Author

@zeroby0 zeroby0 Aug 7, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

resolved in 091f430

This line means the hash for urls will always be the same, even if the remote content changes.

So even if eleventy-cache-assets fetches the new asset, it will not be replaced in the output because the old version with the same hash exists.

queueImage downloads the remote asset so we can use the content for hashing, but statsSync and statsByDimensionsSync will be broken for urls.

Unless

  1. The remote asset content be passed in by the user as options.remoteAssetContent (091f430 implements this). or
  2. We can somehow synchronously fetch assets with eleventy-cache-assets, or
  3. Use the options.__validAssetCache and regenerate images if cache is invalid. But the user will have to treat remote images and local images differently, because remote images are generated from url, not content. For example, such remote images cannot be http cached.

The first option is the most flexible of the three: If the user is unable to pass in the remote content, they can pass in the url as options.remoteAssetContent and the build will not fail. Or they can pass in a random string in production, and be sure that the hash is unique.

}

hash.update(JSON.stringify(opts));
imgHashCache[src] = base64url.encode(hash.digest()).substring(0, length);

return imgHashCache[src];
}

function getFilename(src, width, format, options = {}) {
let id = shorthash(src);
let id = getHash(src, options);

if (typeof options.filenameFormat === "function") {
let filename = options.filenameFormat(id, src, width, format, options);
// if options.filenameFormat returns falsy, use fallback filename
Expand All @@ -147,7 +192,8 @@ function getStats(src, format, urlPath, width, height, options = {}) {
let outputExtension = options.extensions[format] || format;

if(options.urlFormat && typeof options.urlFormat === "function") {
let id = shorthash(src);
let id = getHash(src, options);

url = options.urlFormat({
id,
src,
Expand Down Expand Up @@ -285,6 +331,16 @@ async function resizeImage(src, options = {}) {
let fullStats = getFullStats(src, metadata, options);
for(let outputFormat in fullStats) {
for(let stat of fullStats[outputFormat]) {
if(options.useCache && fs.existsSync(stat.outputPath)){
stat.size = fs.statSync(stat.outputPath).size;
if(options.dryRun) {
stat.buffer = fs.readFileSync(src);
zachleat marked this conversation as resolved.
Show resolved Hide resolved
}

outputFilePromises.push(Promise.resolve(stat));
continue;
}

let sharpInstance = sharpImage.clone();
if(stat.width < metadata.width || (options.svgAllowUpscale && metadata.format === "svg")) {
let resizeOptions = {
Expand Down Expand Up @@ -398,6 +454,8 @@ function queueImage(src, opts) {
// eleventy-cache-assets 2.0.3 and below
input = await assetCache.fetch(cacheOptions);
}

options.remoteAssetContent = input; // Only set for remote assets with URL
} else {
input = src;
}
Expand Down Expand Up @@ -428,11 +486,19 @@ Object.defineProperty(module.exports, "concurrency", {
* the correct location yet.
*/
function statsSync(src, opts) {
if(typeof src === "string" && isFullUrl(src) && !("remoteAssetContent" in opts)) {
throw new Error("When using statsSync or statsByDimensionsSync with URLs, options.remoteAssetContent should be set to the content of the remote asset.");
}

let dimensions = getImageSize(src);
return getFullStats(src, dimensions, opts);
}

function statsByDimensionsSync(src, width, height, opts) {
if(typeof src === "string" && isFullUrl(src) && !("remoteAssetContent" in opts)) {
throw new Error("When using statsSync or statsByDimensionsSync with URLs, options.remoteAssetContent should be set to the content of the remote asset.");
}

let dimensions = { width, height, guess: true };
return getFullStats(src, dimensions, opts);
}
Expand All @@ -441,6 +507,7 @@ module.exports.statsSync = statsSync;
module.exports.statsByDimensionsSync = statsByDimensionsSync;
module.exports.getFormats = getFormatsArray;
module.exports.getWidths = getValidWidths;
module.exports.getHash = getHash;

const generateHTML = require("./generate-html");
module.exports.generateHTML = generateHTML;
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@
"dependencies": {
"@11ty/eleventy-cache-assets": "^2.3.0",
"debug": "^4.3.2",
"base64url": "^3.0.1",
"image-size": "^1.0.0",
"p-queue": "^6.6.2",
"sharp": "^0.29.0",
"short-hash": "^1.0.0"
},
"devDependencies": {
"ava": "^3.15.0",
Expand Down
29 changes: 15 additions & 14 deletions test/test-markup.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ test("Image markup (defaults)", async t => {

t.is(generateHTML(results, {
alt: ""
}), `<picture><source type="image/webp" srcset="/img/97854483-1280.webp 1280w"><img alt="" src="/img/97854483-1280.jpeg" width="1280" height="853"></picture>`);
}), `<picture><source type="image/webp" srcset="/img/Bok0Qhed6a-1280.webp 1280w"><img alt="" src="/img/Bok0Qhed6a-1280.jpeg" width="1280" height="853"></picture>`);
});

test("Image service", async t => {
Expand All @@ -23,7 +23,8 @@ test("Image service", async t => {
widths: [600], // 260-440 in layout
urlFormat: function({ width, format }) {
return `${serviceApiDomain}/api/image/?url=${encodeURIComponent(screenshotUrl)}&width=${width}&format=${format}`;
}
},
remoteAssetContent: 'remote asset content'
};

let results = eleventyImage.statsByDimensionsSync(screenshotUrl, 1440, 900, options);
Expand All @@ -45,13 +46,13 @@ test("Image object (defaults)", async t => {
{
"source": {
type: "image/webp",
srcset: "/img/97854483-1280.webp 1280w",
srcset: "/img/Bok0Qhed6a-1280.webp 1280w",
}
},
{
"img": {
alt: "",
src: "/img/97854483-1280.jpeg",
src: "/img/Bok0Qhed6a-1280.jpeg",
width: 1280,
height: 853,
}
Expand All @@ -70,9 +71,9 @@ test("Image markup (two widths)", async t => {
alt: "",
sizes: "100vw",
}), [`<picture>`,
`<source type="image/webp" srcset="/img/97854483-200.webp 200w, /img/97854483-400.webp 400w" sizes="100vw">`,
`<source type="image/jpeg" srcset="/img/97854483-200.jpeg 200w, /img/97854483-400.jpeg 400w" sizes="100vw">`,
`<img alt="" src="/img/97854483-200.jpeg" width="400" height="266">`,
`<source type="image/webp" srcset="/img/Bok0Qhed6a-200.webp 200w, /img/Bok0Qhed6a-400.webp 400w" sizes="100vw">`,
`<source type="image/jpeg" srcset="/img/Bok0Qhed6a-200.jpeg 200w, /img/Bok0Qhed6a-400.jpeg 400w" sizes="100vw">`,
`<img alt="" src="/img/Bok0Qhed6a-200.jpeg" width="400" height="266">`,
`</picture>`].join(""));
});

Expand All @@ -95,7 +96,7 @@ test("Image markup (two formats)", async t => {

t.is(generateHTML(results, {
alt: ""
}), `<picture><source type="image/avif" srcset="/img/97854483-1280.avif 1280w"><img alt="" src="/img/97854483-1280.webp" width="1280" height="853"></picture>`);
}), `<picture><source type="image/avif" srcset="/img/Bok0Qhed6a-1280.avif 1280w"><img alt="" src="/img/Bok0Qhed6a-1280.webp" width="1280" height="853"></picture>`);
});

test("Image markup (one format)", async t => {
Expand All @@ -107,7 +108,7 @@ test("Image markup (one format)", async t => {
t.is(generateHTML(results, {
alt: "",
sizes: "100vw"
}), `<img alt="" src="/img/97854483-1280.jpeg" width="1280" height="853">`);
}), `<img alt="" src="/img/Bok0Qhed6a-1280.jpeg" width="1280" height="853">`);
});

test("Image markup (auto format)", async t => {
Expand All @@ -119,7 +120,7 @@ test("Image markup (auto format)", async t => {
t.is(generateHTML(results, {
alt: "",
sizes: "100vw"
}), `<img alt="" src="/img/97854483-1280.jpeg" width="1280" height="853">`);
}), `<img alt="" src="/img/Bok0Qhed6a-1280.jpeg" width="1280" height="853">`);
});

test("Image markup (one format, two widths)", async t => {
Expand All @@ -132,7 +133,7 @@ test("Image markup (one format, two widths)", async t => {
t.is(generateHTML(results, {
alt: "",
sizes: "100vw"
}), `<img alt="" src="/img/97854483-100.jpeg" width="200" height="133" srcset="/img/97854483-100.jpeg 100w, /img/97854483-200.jpeg 200w" sizes="100vw">`);
}), `<img alt="" src="/img/Bok0Qhed6a-100.jpeg" width="200" height="133" srcset="/img/Bok0Qhed6a-100.jpeg 100w, /img/Bok0Qhed6a-200.jpeg 200w" sizes="100vw">`);
});

test("Image markup (throws on invalid object)", async t => {
Expand Down Expand Up @@ -160,8 +161,8 @@ test("Image markup (defaults, inlined)", async t => {
}, {
whitespaceMode: "block"
}), `<picture>
<source type="image/webp" srcset="/img/97854483-1280.webp 1280w">
<img alt="" src="/img/97854483-1280.jpeg" width="1280" height="853">
<source type="image/webp" srcset="/img/Bok0Qhed6a-1280.webp 1280w">
<img alt="" src="/img/Bok0Qhed6a-1280.jpeg" width="1280" height="853">
</picture>`);
});

Expand All @@ -175,7 +176,7 @@ test("svgShortCircuit and generateHTML: Issue #48", async t => {
let html = eleventyImage.generateHTML(stats, {
alt: "Tiger",
});
t.is(html, `<img alt="Tiger" src="/img/8b4d670b-900.svg" width="900" height="900">`);
t.is(html, `<img alt="Tiger" src="/img/VBWwBwQG9D-900.svg" width="900" height="900">`);
});

test("Filter out empty format arrays", async t => {
Expand Down
45 changes: 23 additions & 22 deletions test/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ test("Use 'auto' format as original", async t => {

t.is(stats.auto, undefined);
t.is(stats.jpeg.length, 1);
t.is(stats.jpeg[0].outputPath, path.join("test/img/97854483-1280.jpeg"));
t.is(stats.jpeg[0].outputPath, path.join("test/img/Bok0Qhed6a-1280.jpeg"));
t.is(stats.jpeg[0].width, 1280);
});

Expand All @@ -181,7 +181,7 @@ test("Try to use a width larger than original", async t => {
outputDir: "./test/img/"
});
t.is(stats.jpeg.length, 1);
t.is(stats.jpeg[0].outputPath, path.join("test/img/97854483-1280.jpeg"));
t.is(stats.jpeg[0].outputPath, path.join("test/img/Bok0Qhed6a-1280.jpeg"));
t.is(stats.jpeg[0].width, 1280);
});

Expand All @@ -192,7 +192,7 @@ test("Try to use a width larger than original (two sizes)", async t => {
outputDir: "./test/img/"
});
t.is(stats.jpeg.length, 1);
t.is(stats.jpeg[0].outputPath, path.join("test/img/97854483-1280.jpeg"));
t.is(stats.jpeg[0].outputPath, path.join("test/img/Bok0Qhed6a-1280.jpeg"));
t.is(stats.jpeg[0].width, 1280);
});

Expand All @@ -203,7 +203,7 @@ test("Try to use a width larger than original (with a null in there)", async t =
outputDir: "./test/img/"
});
t.is(stats.jpeg.length, 1);
t.is(stats.jpeg[0].outputPath, path.join("test/img/97854483-1280.jpeg"));
t.is(stats.jpeg[0].outputPath, path.join("test/img/Bok0Qhed6a-1280.jpeg"));
t.is(stats.jpeg[0].width, 1280);
});

Expand All @@ -214,7 +214,7 @@ test("Just falsy width", async t => {
outputDir: "./test/img/"
});
t.is(stats.jpeg.length, 1);
t.is(stats.jpeg[0].outputPath, path.join("test/img/97854483-1280.jpeg"));
t.is(stats.jpeg[0].outputPath, path.join("test/img/Bok0Qhed6a-1280.jpeg"));
t.is(stats.jpeg[0].width, 1280);
});

Expand All @@ -226,7 +226,7 @@ test("Use exact same width as original", async t => {
});
t.is(stats.jpeg.length, 1);
// breaking change in 0.5: always use width in filename
t.is(stats.jpeg[0].outputPath, path.join("test/img/97854483-1280.jpeg"));
t.is(stats.jpeg[0].outputPath, path.join("test/img/Bok0Qhed6a-1280.jpeg"));
t.is(stats.jpeg[0].width, 1280);
});

Expand All @@ -237,7 +237,7 @@ test("Try to use a width larger than original (statsSync)", t => {
});

t.is(stats.jpeg.length, 1);
t.is(stats.jpeg[0].url, "/img/97854483-1280.jpeg");
t.is(stats.jpeg[0].url, "/img/Bok0Qhed6a-1280.jpeg");
t.is(stats.jpeg[0].width, 1280);
});

Expand All @@ -248,7 +248,7 @@ test("Use exact same width as original (statsSync)", t => {
});

t.is(stats.jpeg.length, 1);
t.is(stats.jpeg[0].url, "/img/97854483-1280.jpeg"); // no width in filename
t.is(stats.jpeg[0].url, "/img/Bok0Qhed6a-1280.jpeg"); // no width in filename
t.is(stats.jpeg[0].width, 1280);
});

Expand All @@ -270,19 +270,20 @@ test("Use custom function to define file names", async (t) => {
});

t.is(stats.jpeg.length, 2);
t.is(stats.jpeg[0].outputPath, path.join("test/img/bio-2017-97854483-600.jpeg"));
t.is(stats.jpeg[0].url, "/img/bio-2017-97854483-600.jpeg");
t.is(stats.jpeg[0].srcset, "/img/bio-2017-97854483-600.jpeg 600w");
t.is(stats.jpeg[0].outputPath, path.join("test/img/bio-2017-Bok0Qhed6a-600.jpeg"));
t.is(stats.jpeg[0].url, "/img/bio-2017-Bok0Qhed6a-600.jpeg");
t.is(stats.jpeg[0].srcset, "/img/bio-2017-Bok0Qhed6a-600.jpeg 600w");
t.is(stats.jpeg[0].width, 600);
t.is(stats.jpeg[1].outputPath, path.join("test/img/bio-2017-97854483-1280.jpeg"));
t.is(stats.jpeg[1].url, "/img/bio-2017-97854483-1280.jpeg");
t.is(stats.jpeg[1].srcset, "/img/bio-2017-97854483-1280.jpeg 1280w");
t.is(stats.jpeg[1].outputPath, path.join("test/img/bio-2017-Bok0Qhed6a-1280.jpeg"));
t.is(stats.jpeg[1].url, "/img/bio-2017-Bok0Qhed6a-1280.jpeg");
t.is(stats.jpeg[1].srcset, "/img/bio-2017-Bok0Qhed6a-1280.jpeg 1280w");
t.is(stats.jpeg[1].width, 1280);
});

test("Unavatar test", t => {
let stats = eleventyImage.statsByDimensionsSync("https://unavatar.now.sh/twitter/zachleat?fallback=false", 400, 400, {
widths: [75]
widths: [75],
remoteAssetContent: 'remote asset content'
});

t.is(stats.webp.length, 1);
Expand Down Expand Up @@ -386,7 +387,7 @@ test("Sync by dimension with jpeg input (wrong dimensions, supplied are smaller

// this won’t upscale so it will miss out on higher resolution images but there won’t be any broken image URLs in the output
t.is(stats.jpeg.length, 1);
t.is(stats.jpeg[0].outputPath, path.join("img/97854483-164.jpeg"));
t.is(stats.jpeg[0].outputPath, path.join("img/Bok0Qhed6a-164.jpeg"));
});

test("Sync by dimension with jpeg input (wrong dimensions, supplied are larger than real)", t => {
Expand All @@ -396,8 +397,8 @@ test("Sync by dimension with jpeg input (wrong dimensions, supplied are larger t
});

t.is(stats.jpeg.length, 2);
t.is(stats.jpeg[0].outputPath, path.join("img/97854483-164.jpeg"));
t.is(stats.jpeg[1].outputPath, path.join("img/97854483-328.jpeg"));
t.is(stats.jpeg[0].outputPath, path.join("img/Bok0Qhed6a-164.jpeg"));
t.is(stats.jpeg[1].outputPath, path.join("img/Bok0Qhed6a-328.jpeg"));
});

test("Keep a cache, reuse with same file names and options", async t => {
Expand Down Expand Up @@ -545,14 +546,14 @@ test("Using `jpg` in formats Issue #64", async t => {
t.deepEqual(stats, {
jpeg: [
{
filename: '97854483-1280.jpeg',
filename: 'Bok0Qhed6a-1280.jpeg',
format: 'jpeg',
height: 853,
outputPath: path.join('img/97854483-1280.jpeg'),
outputPath: path.join('img/Bok0Qhed6a-1280.jpeg'),
size: 276231,
sourceType: "image/jpeg",
srcset: '/img/97854483-1280.jpeg 1280w',
url: '/img/97854483-1280.jpeg',
srcset: '/img/Bok0Qhed6a-1280.jpeg 1280w',
url: '/img/Bok0Qhed6a-1280.jpeg',
width: 1280,
},
]
Expand Down