-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(ci): add sitemap uploader (#259)
* feat(ci): add sitemap uploader * test(ci): fix tests * fix(sitemap): handle BASE_URL replacement * fix: make job multiple * fix: use curl, better BASE_URL replacement * chore(ci): add Generate sitemap (prod) job * Update .gitlab-ci.yml * Update .gitlab-ci.yml * fix: curl exitError when fail * fix: add allow_failure * chore(ci): use internal service url to fetch sitemap * chore(ci): use real url * fix: fix namespace * ci: add create sitemap dev * Update HOWTO.md Co-authored-by: Julien Bouquillon <[email protected]> * Apply suggestions from code review Co-authored-by: Julien Bouquillon <[email protected]> * Apply suggestions from code review * Update .gitlab-ci.yml Co-authored-by: Julien Bouquillon <[email protected]> * ci: filter manul job * ci: move manual job to .post stage * ci: manual job don't have to wait for jobs * ci: fix manual * fix: ci Co-authored-by: LionelB <[email protected]> Co-authored-by: Lionel <[email protected]>
- Loading branch information
3 people
authored
Jan 22, 2021
1 parent
7897f1f
commit f3a87ea
Showing
17 changed files
with
241 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
58 changes: 58 additions & 0 deletions
58
.k8s/__tests__/__snapshots__/generate-prod-sitemap-uploader.ts.snap
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| // Jest Snapshot v1, https://goo.gl/fbAQLP | ||
|
|
||
| exports[`kosko generate --prod sitemap-uploader 1`] = ` | ||
| "metadata: | ||
| name: sitemap-uploader-424242 | ||
| namespace: cdtn-admin-secret | ||
| spec: | ||
| backoffLimit: 3 | ||
| template: | ||
| metadata: | ||
| name: sitemap-uploader-424242 | ||
| namespace: cdtn-admin-secret | ||
| spec: | ||
| restartPolicy: OnFailure | ||
| containers: | ||
| - name: az-sitemap-uploader | ||
| image: 'mcr.microsoft.com/azure-cli:2.9.1' | ||
| command: | ||
| - bash | ||
| args: | ||
| - '-c' | ||
| - |+ | ||
| echo \\"Fetch sitemap from $SITEMAP_ENDPOINT\\" | ||
| curl --fail -L $SITEMAP_ENDPOINT -o sitemap.xml | ||
| if [[ -f sitemap.xml ]]; then | ||
| # replace the default urls hostname if $BASE_URL is given | ||
| [[ -n $BASE_URL ]] && sed -i -E 's#<loc>https?://[^/]*/#<loc>'$BASE_URL'/#' sitemap.xml | ||
| # upload | ||
| echo \\"Upload sitemap to azure/$DESTINATION_CONTAINER/$DESTINATION_NAME\\" | ||
| az storage blob upload --account-name $AZ_ACCOUNT_NAME --account-key $AZ_ACCOUNT_KEY --container-name $DESTINATION_CONTAINER --file sitemap.xml --name $DESTINATION_NAME | ||
| else | ||
| echo \\"Cannot fetch sitemap.xml, abort\\" | ||
| exit 1 | ||
| fi | ||
| env: | ||
| - name: DESTINATION_CONTAINER | ||
| value: destination-container | ||
| - name: DESTINATION_NAME | ||
| value: sitemap.xml | ||
| - name: SITEMAP_ENDPOINT | ||
| value: 'https://path/to/sitemap' | ||
| - name: BASE_URL | ||
| value: 'https://host.tmp' | ||
| - name: AZ_ACCOUNT_NAME | ||
| value: $(azurestorageaccountname) | ||
| - name: AZ_ACCOUNT_KEY | ||
| value: $(azurestorageaccountkey) | ||
| envFrom: | ||
| - secretRef: | ||
| name: azure-volume-dev-secret | ||
| apiVersion: batch/v1 | ||
| kind: Job | ||
| " | ||
| `; |
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // | ||
|
|
||
| import { getEnvManifests } from "@socialgouv/kosko-charts/testing"; | ||
| import { project } from "@socialgouv/kosko-charts/testing/fake/gitlab-ci.env"; | ||
|
|
||
| jest.setTimeout(1000 * 60); | ||
| test("kosko generate --prod sitemap-uploader", async () => { | ||
| expect( | ||
| await getEnvManifests("prod", "sitemap-uploader", { | ||
| ...project("cdtn-admin").prod, | ||
| SITEMAP_ENDPOINT: "https://path/to/sitemap", | ||
| DESTINATION_CONTAINER: "destination-container", | ||
| DESTINATION_NAME: "sitemap.xml", | ||
| SECRET_NAME: "azure-volume-dev-secret", | ||
| BASE_URL: "https://host.tmp", | ||
| }) | ||
| ).toMatchSnapshot(); | ||
| }); |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| import { ok } from "assert"; | ||
| import { Job } from "kubernetes-models/batch/v1/Job"; | ||
|
|
||
| // renovate: datasource=docker depName=mcr.microsoft.com/azure-cli versioning=2.9.1 | ||
| const AZ_DOCKER_TAG = "2.9.1"; | ||
|
|
||
| const uploadSitemapScript = ` | ||
| echo "Fetch sitemap from $SITEMAP_ENDPOINT" | ||
| curl --fail -L $SITEMAP_ENDPOINT -o sitemap.xml | ||
| if [[ -f sitemap.xml ]]; then | ||
| # replace the default urls hostname if $BASE_URL is given | ||
| [[ -n $BASE_URL ]] && sed -i -E 's#<loc>https?://[^/]*/#<loc>'$BASE_URL'/#' sitemap.xml | ||
| # upload | ||
| echo "Upload sitemap to azure/$DESTINATION_CONTAINER/$DESTINATION_NAME" | ||
| az storage blob upload --account-name $AZ_ACCOUNT_NAME --account-key $AZ_ACCOUNT_KEY --container-name $DESTINATION_CONTAINER --file sitemap.xml --name $DESTINATION_NAME | ||
| else | ||
| echo "Cannot fetch sitemap.xml, abort" | ||
| exit 1 | ||
| fi | ||
| `; | ||
|
|
||
| ok(process.env.SITEMAP_ENDPOINT); // https://url/sitemap | ||
| ok(process.env.DESTINATION_CONTAINER); // sitemap | ||
| ok(process.env.DESTINATION_NAME); // sitemap.xml | ||
| ok(process.env.SECRET_NAME); // azure-cdtnadmindev-volume | azure-cdtnadminprod-volume | ||
|
|
||
| const createSitemapJob = () => { | ||
| const job = new Job({ | ||
| metadata: { | ||
| name: `sitemap-uploader-${process.env.CI_JOB_ID}`, | ||
| namespace: "cdtn-admin-secret", | ||
| }, | ||
|
|
||
| spec: { | ||
| backoffLimit: 3, | ||
| template: { | ||
| metadata: { | ||
| name: `sitemap-uploader-${process.env.CI_JOB_ID}`, | ||
| namespace: "cdtn-admin-secret", | ||
| }, | ||
| spec: { | ||
| restartPolicy: "OnFailure", | ||
| containers: [ | ||
| { | ||
| name: "az-sitemap-uploader", | ||
| image: `mcr.microsoft.com/azure-cli:${AZ_DOCKER_TAG}`, | ||
| command: ["bash"], | ||
| args: ["-c", uploadSitemapScript], | ||
| env: [ | ||
| { | ||
| name: "DESTINATION_CONTAINER", | ||
| value: process.env.DESTINATION_CONTAINER, | ||
| }, | ||
| { | ||
| name: "DESTINATION_NAME", | ||
| value: process.env.DESTINATION_NAME, | ||
| }, | ||
| { | ||
| name: "SITEMAP_ENDPOINT", | ||
| value: process.env.SITEMAP_ENDPOINT, | ||
| }, | ||
| { | ||
| // should not contain ending slash | ||
| name: "BASE_URL", | ||
| value: process.env.BASE_URL, | ||
| }, | ||
| { | ||
| name: "AZ_ACCOUNT_NAME", | ||
| value: "$(azurestorageaccountname)", | ||
| }, | ||
| { | ||
| name: "AZ_ACCOUNT_KEY", | ||
| value: "$(azurestorageaccountkey)", | ||
| }, | ||
| ], | ||
| envFrom: [ | ||
| { | ||
| secretRef: { | ||
| name: process.env.SECRET_NAME, | ||
| }, | ||
| }, | ||
| ], | ||
| }, | ||
| ], | ||
| }, | ||
| }, | ||
| }, | ||
| }); | ||
| return job; | ||
| }; | ||
|
|
||
| const job = createSitemapJob(); | ||
| export default job; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| # Comment on fait pour... ? | ||
|
|
||
| - [Créer un fichier sitemap.xml](#Créer-un-fichier-sitemap.xml) | ||
|
|
||
| ## Créer un fichier sitemap-branch.xml | ||
|
|
||
| Lancer la tâche `Generate Sitemap (dev)` en spécifiant les paramètres | ||
|
|
||
| ```sh | ||
| SITEMAP_ENDPOINT: "https://cdtn-admin.fabrique.social.gouv.fr/api/sitemap" | ||
| DESTINATION_NAME: sitemap-branch.xml | ||
| ``` |