feat: fetch remote images (#368)

.github/workflows/ci.yml

@@ -43,6 +43,14 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-hugomod-
 
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: 'v1.x'
+
+      - run: make fetchdata
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - run: make cibuild
 
       - name: Deploy to Netlify
@@ -61,8 +69,6 @@ jobs:
         timeout-minutes: 1
 
       - run: make cibuild-prod
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Deploy
         uses: peaceiris/[email protected]

Makefile

@@ -67,8 +67,13 @@ cibuild:
 .PHONY: cibuild-prod
 cibuild-prod:
 	cd ./exampleSite && \
-		bash ./scripts/fetch_data.sh ${GH_USER_ID} > ./data/github/${GH_USER_ID}.json && \
 		hugo --minify --cleanDestinationDir \
 			--baseURL ${BASE_URL} \
 			--i18n-warnings --path-warnings && \
 		wget -O ./public/report.html ${BASE_URL}/report.html || true
+
+.PHONY: fetchdata
+fetchdata:
+	cd ./exampleSite && \
+		bash ./scripts/fetch_data.sh ${GH_USER_ID} > ./data/github/${GH_USER_ID}.json && \
+		deno run --allow-net --allow-read --allow-write --unstable scripts/fetch_images.ts

exampleSite/assets/images/.gitkeep

@@ -0,0 +1 @@
+keep this directory

exampleSite/content/en/posts/github-pages-and-github-actions.md

@@ -3,7 +3,7 @@ title: GitHub Pages and GitHub Actions
 description: Deploy your static site to GitHub Pages using GitHub Actions
 publishdate: 2019-11-01T01:00:00.000Z
 # draft: true
-eyecatch: images/github-pages-and-github-actions.jpg
+eyecatch: 'https://user-images.githubusercontent.com/30958501/121798925-0c4b7600-cc64-11eb-89eb-92d2e8f746db.jpg'
 # tags: ["GitHub Actions"]
 toc: true
 # math: true

exampleSite/content/en/posts/math.md

@@ -3,7 +3,7 @@ title: MathJax Support $E=mc^2$
 description: Beautiful math in all browsers $E=mc^2$
 publishdate: 2020-01-01T01:00:00.000Z
 # draft: true
-# eyecatch: images/mathjax.jpg
+eyecatch: "https://user-images.githubusercontent.com/30958501/121796747-647b7b80-cc56-11eb-8490-52d5899e561c.jpg"
 # tags: ["hugo"]
 toc: true
 math: true

exampleSite/content/ja/posts/github-pages-and-github-actions.md

@@ -3,7 +3,7 @@ title: GitHub Pages and GitHub Actions
 description: GitHub Actions による GitHub Pages へのデプロイ
 publishdate: 2019-11-01T01:00:00.000Z
 # draft: true
-eyecatch: images/github-pages-and-github-actions.jpg
+eyecatch: 'https://user-images.githubusercontent.com/30958501/121798925-0c4b7600-cc64-11eb-89eb-92d2e8f746db.jpg'
 # tags: ["GitHub Actions"]
 toc: true
 # math: true

exampleSite/scripts/fetch_images.ts

@@ -0,0 +1,108 @@
+import { expandGlobSync } from "https://deno.land/[email protected]/fs/mod.ts";
+import {
+  createHash,
+  SupportedAlgorithm,
+} from "https://deno.land/[email protected]/hash/mod.ts";
+import {
+  Destination,
+  download,
+  DownlodedFile,
+} from "https://deno.land/x/[email protected]/mod.ts";
+import { exists } from "https://deno.land/[email protected]/fs/mod.ts";
+
+class Image {
+  remoteUrl: string;
+  remoteUrlHashed: string;
+  saveDir: string;
+  extension: string;
+
+  constructor(remoteUrl: string, saveDir: string) {
+    this.remoteUrl = remoteUrl;
+    this.remoteUrlHashed = this.hash("sha256", remoteUrl);
+    this.saveDir = saveDir;
+    this.extension = this.getExtension(this.remoteUrl);
+  }
+
+  hash(algorithm: SupportedAlgorithm, str: string): string {
+    const hash = createHash(`${algorithm}`);
+    hash.update(str);
+    return hash.toString();
+  }
+
+  getExtension(filename: string) {
+    return filename.split(".").pop() || "";
+  }
+
+  async download() {
+    const dst: Destination = {
+      file: `${this.remoteUrlHashed}.${this.extension}`,
+      dir: this.saveDir,
+    };
+
+    if (await exists(`${dst.dir}/${dst.file}`)) {
+      console.log(`info: skip ${this.remoteUrl}`);
+      return;
+    }
+
+    try {
+      const fileObj: DownlodedFile = await download(this.remoteUrl, dst);
+      console.log(`info: donwload ${this.remoteUrl} to ${fileObj.fullPath}`);
+    } catch (err) {
+      console.log(err);
+    }
+  }
+}
+
+class Post {
+  path: string;
+  images: Array<Image>;
+  imageSaveDir: string;
+
+  constructor(path: string, imageSaveDir: string) {
+    this.path = path;
+    this.images = [];
+    this.imageSaveDir = imageSaveDir;
+  }
+
+  async getImageUrls() {
+    const markdownContent = await Deno.readTextFile(this.path);
+    // https://docs.github.com/en/github/managing-your-work-on-github/file-attachments-on-issues-and-pull-requests
+    const regexMarkdownImageLinks =
+      /https:\/\/user-images.githubusercontent.com\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+\.(png|gif|jpg|jpeg|mp4|mov)/gm;
+    const matches = markdownContent.match(regexMarkdownImageLinks);
+    if (matches === null) return;
+    for (const remoteUrl of matches) {
+      const image = new Image(remoteUrl, this.imageSaveDir);
+      this.images.push(image);
+    }
+  }
+}
+
+class Blog {
+  domain: string;
+  posts: Array<Post>;
+  imageSaveDir: string;
+
+  constructor(domain: string, imageSaveDir: string) {
+    this.domain = domain;
+    this.posts = [];
+    this.imageSaveDir = imageSaveDir;
+  }
+
+  getPosts(glob: string) {
+    for (const file of expandGlobSync(glob)) {
+      const post = new Post(file.path, this.imageSaveDir);
+      this.posts.push(post);
+    }
+  }
+}
+
+const blog = new Blog("peaceiris.com", "./assets/images");
+await blog.getPosts("./content/**/*.md");
+
+for await (const post of blog.posts) {
+  await post.getImageUrls();
+  for await (const image of post.images) {
+    image.download();
+  }
+}

layouts/_default/_markup/render-image.html

@@ -3,7 +3,15 @@
   {{ $caption = . | safeHTML }}
 {{ end }}
 
-{{ with resources.Get (printf "%s" ($.Destination | safeURL)) }}
+{{ $hashedDestination := sha256 $.Destination }}
+{{ $extension := path.Ext $.Destination }}
+{{ $localPath := delimit (slice "images/" $hashedDestination $extension) "" }}
+{{ $imagePath := $.Destination }}
+{{ if (fileExists $localPath) }}
+  {{ $imagePath = $localPath }}
+{{ end }}
+
+{{ with resources.Get $imagePath }}
   {{ $image := . }}
   {{ $small := . }}
   {{ $medium := . }}
@@ -31,7 +39,7 @@
 {{ else }}
   <figure class="image container">
     <img
-      src="{{ $.Destination }}"
+      src="{{ $imagePath }}"
       alt="{{ $caption }}"
       decoding="async"
       loading="lazy"

layouts/partials/eyecatch.html

@@ -1,28 +1,49 @@
-{{ if .Params.eyecatch }}
-  {{ $image := resources.Get (printf "%s" ($.Params.eyecatch | safeURL)) }}
-  {{ $small := $image }}
-  {{ $medium := $image }}
-  {{ $large := $image }}
-  {{ if hugo.IsExtended }}
-    {{ $small = $image.Resize "480x webp" }}
-    {{ $medium = $image.Resize "768x webp" }}
-    {{ $large = $image.Resize "1024x webp" }}
-  {{ else }}
-    {{ $small = $image.Resize "480x jpg" }}
-    {{ $medium = $image.Resize "768x jpg" }}
-    {{ $large = $image.Resize "1024x jpg" }}
+{{ with $.Params.eyecatch }}
+
+  {{ $hashedDestination := sha256 $.Params.eyecatch }}
+  {{ $extension := path.Ext $.Params.eyecatch }}
+  {{ $localPath := delimit (slice "images/" $hashedDestination $extension) "" }}
+  {{ $assetPath := delimit (slice "assets/" $localPath) "" }}
+  {{ $imagePath := $.Params.eyecatch }}
+  {{ if (fileExists $assetPath) }}
+    {{ $imagePath = $localPath }}
   {{ end }}
-  <figure class="image container">
-    <picture>
-      <source media="(max-width: 480px)" srcset="{{ $small.RelPermalink }} 480w">
-      <source media="(max-width: 768px)" srcset="{{ $medium.RelPermalink }} 768w">
-      <source media="(max-width: 1024px)" srcset="{{ $large.RelPermalink }} 1024w">
+
+  {{ with resources.Get $imagePath }}
+    {{ $image := . }}
+    {{ $small := $image }}
+    {{ $medium := $image }}
+    {{ $large := $image }}
+    {{ if hugo.IsExtended }}
+      {{ $small = $image.Resize "480x webp" }}
+      {{ $medium = $image.Resize "768x webp" }}
+      {{ $large = $image.Resize "1024x webp" }}
+    {{ else }}
+      {{ $small = $image.Resize "480x jpg" }}
+      {{ $medium = $image.Resize "768x jpg" }}
+      {{ $large = $image.Resize "1024x jpg" }}
+    {{ end }}
+    <figure class="image container">
+      <picture>
+        <source media="(max-width: 480px)" srcset="{{ $small.RelPermalink }} 480w">
+        <source media="(max-width: 768px)" srcset="{{ $medium.RelPermalink }} 768w">
+        <source media="(max-width: 1024px)" srcset="{{ $large.RelPermalink }} 1024w">
+        <img
+          src="{{ $large.RelPermalink }}"
+          alt="{{ $.Params.Description }}"
+          decoding="async"
+          loading="lazy"
+        />
+      </picture>
+    </figure>
+  {{ else }}
+    <figure class="image container">
       <img
-        src="{{ $large.RelPermalink }}"
+        src="{{ $imagePath }}"
         alt="{{ $.Params.Description }}"
         decoding="async"
         loading="lazy"
       />
-    </picture>
-  </figure>
+    </figure>
+  {{ end }}
 {{ end }}

layouts/partials/head/ogp.html

@@ -1,7 +1,11 @@
 {{ $imagePath := "" }}
 {{ if $.Params.eyecatch }}
-  {{ $image := resources.Get $.Params.eyecatch }}
-  {{ $imagePath = ($image | resources.Fingerprint "sha512").Permalink }}
+  {{ with resources.Get $.Params.eyecatch }}
+    {{ $image := . }}
+    {{ $imagePath = ($image | resources.Fingerprint "sha512").Permalink }}
+  {{ else }}
+    {{ $imagePath = $.Params.eyecatch }}
+  {{ end }}
 {{ else }}
   {{ $imagePath = $.Site.Params.ogp.image | absURL }}
 {{ end }}