From fc959afcf4c65ce4e1d026ec0ac0dbb9bbb06cd7 Mon Sep 17 00:00:00 2001 From: Michal Kimle Date: Tue, 25 Oct 2022 09:23:03 +0200 Subject: [PATCH] Add Docker image publishing to packaging scripts --- .github/workflows/build-test.yml | 18 +-- docker/README.md | 58 ++++++++-- docker/scripts/cli.ts | 104 +++++++++++------- .../{build-docker-images.ts => docker.ts} | 20 ++++ docker/scripts/utils.ts | 6 +- package.json | 12 +- 6 files changed, 155 insertions(+), 63 deletions(-) rename docker/scripts/{build-docker-images.ts => docker.ts} (65%) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 4e31da1617..b70f54671d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -129,20 +129,14 @@ jobs: - name: Publish NPM packages to local NPM registry run: yarn docker:scripts:publish-packages --npm-registry local --npm-tag ${{ github.sha }} --snapshot - name: Build Docker containers - run: yarn docker:scripts:build-docker-images --dev --npm-registry local --npm-tag ${{ github.sha }} --docker-tag ${{ github.sha }} + run: yarn docker:scripts:docker:build --dev --npm-registry local --npm-tag ${{ github.sha }} --docker-tag ${{ github.sha }} - name: Stop local NPM registry run: yarn docker:scripts:npm-registry:stop - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Push airnode-admin to Docker Hub - run: docker push api3/airnode-admin-dev:${{ github.sha }} - - name: Push airnode-client to Docker Hub - run: docker push api3/airnode-client-dev:${{ github.sha }} - - name: Push airnode-deployer to Docker Hub - run: docker push api3/airnode-deployer-dev:${{ github.sha }} + - name: Publish Docker containers + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: yarn docker:scripts:docker:publish --dev --docker-tag ${{ github.sha }} unit-tests: name: Unit tests runs-on: ubuntu-latest diff --git a/docker/README.md b/docker/README.md index 99b7930440..9810e58cfc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -3,8 +3,8 @@ This is a Docker container that can: - Start/stop a local NPM registry Docker container -- Build and publish NPM packages to both local and official (TODO) NPM registry -- Build Docker containers from both local and official NPM packages +- Build and publish NPM packages to both local and official NPM registry +- Build and publish Docker containers from both local and official NPM packages The container uses so called Docker-in-Docker method to build packages and Docker container in the clean Dockerized environment. @@ -27,7 +27,7 @@ There are three CLI commands available: - [`npm-registry`](#npm-registry) - [`publish-packages`](#publish-packages) -- [`build-docker-images`](#build-docker-images) +- [`docker`](#docker) **To run all the pieces together and build Docker images, you can use the two convenience Yarn targets:** @@ -92,19 +92,26 @@ Use the `--npm-tag` option to specify the tag for the published packages. Use the `--snapshot` option to publish a [snapshot package](https://github.com/changesets/changesets/blob/main/docs/snapshot-releases.md) +When publishing to the official NPM registry you have to provide `NPM_TOKEN` environment variable containing NPM +registry authentication token. + Example: ```bash docker run --rm -v $(pwd):/airnode -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest publish-packages --npm-registry local --npm-tag local --snapshot ``` -You can use a convenience Yarn target to publish snapshot packages to a local NPM registry: +You can use two convenience Yarn targets to publish snapshot packages to a local or official NPM registry (snapshots +only at the moment): ```bash yarn docker:scripts:publish-packages:local +yarn docker:scripts:publish-packages:snapshot ``` -### build-docker-images +### docker + +**build** ``` Build Docker images @@ -134,13 +141,48 @@ Use the `--dev` option to build the development images, with the `-dev` suffix i Example: ```bash -docker run --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest build-docker-images --npm-registry local --npm-tag local --docker-tag local +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest docker build --npm-registry local --npm-tag local --docker-tag local ``` You can use two convenience Yarn targets for building Docker images from the local NPM packages and from the latest official ones: ```bash -yarn docker:scripts:build-docker-images:local -yarn docker:scripts:build-docker-images:latest +yarn docker:scripts:docker:build:local +yarn docker:scripts:docker:build:latest +``` + +**publish** + +``` +Publish Docker images + +Options: + --version Show version number [boolean] + --help Show help [boolean] + -g, --docker-tag Docker tag to build the images under [string] [default: "latest"] + -d, --dev Build Docker dev images (with -dev suffix) [boolean] [default: false] +``` + +You can publish (push) Airnode Docker images. + +Use the `--docker-tag` option to specify the Docker tag of the images that should be pushed. + +Use the `--dev` option to push the images, with the `-dev` suffix in their name. + +You need to provide two environment variables to authenticate against the DockerHub registry: + +- `DOCKERHUB_USERNAME` - DockerHub username +- `DOCKERHUB_TOKEN` - DockerHub access token + +Example: + +```bash +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e DOCKERHUB_USERNAME -e DOCKERHUB_TOKEN api3/airnode-packaging:latest docker publish +``` + +You can use a convenience Yarn target for publishing the latest Docker images: + +```bash +docker:scripts:docker:publish:latest ``` diff --git a/docker/scripts/cli.ts b/docker/scripts/cli.ts index 64f6407596..3a1a79a217 100644 --- a/docker/scripts/cli.ts +++ b/docker/scripts/cli.ts @@ -3,7 +3,7 @@ import omitBy from 'lodash/omitBy'; import { logger } from '@api3/airnode-utilities'; import { go, GoResult, goSync } from '@api3/promise-utils'; import { stopNpmRegistry, startNpmRegistry } from './npm-registry'; -import { buildDockerImages } from './build-docker-images'; +import { buildDockerImages, publishDockerImages } from './docker'; import { publishPackages } from './publish-packages'; // Taken from airnode-deployer @@ -88,42 +88,72 @@ yargs(process.argv.slice(2)) }); } ) - .command( - 'build-docker-images', - 'Build Docker images', - { - 'npm-registry': { - alias: 'r', - description: 'NPM registry URL to fetch packages from or a keyword `local` to use a local NPM registry', - default: 'https://registry.npmjs.org/', - type: 'string', - }, - 'npm-tag': { - alias: 't', - description: 'NPM tag/version of the packages that will be fetched', - default: 'latest', - type: 'string', - }, - 'docker-tag': { - alias: 'g', - description: 'Docker tag to build the images under', - default: 'latest', - type: 'string', - }, - dev: { - alias: 'd', - description: 'Build Docker dev images (with -dev suffix)', - default: false, - type: 'boolean', - }, - }, - (args) => { - logger.log(`Running command '${args._[0]}' with arguments ${longArguments(args)}`); - runCliCommand(() => { - buildDockerImages(args.npmRegistry, args.npmTag, args.dockerTag, args.dev); - }); - } - ) + .command('docker', 'Manages Docker images', (yargs) => { + yargs + .command( + 'build', + 'Build Docker images', + { + 'npm-registry': { + alias: 'r', + description: 'NPM registry URL to fetch packages from or a keyword `local` to use a local NPM registry', + default: 'https://registry.npmjs.org/', + type: 'string', + }, + 'npm-tag': { + alias: 't', + description: 'NPM tag/version of the packages that will be fetched', + default: 'latest', + type: 'string', + }, + 'docker-tag': { + alias: 'g', + description: 'Docker tag to build the images under', + default: 'latest', + type: 'string', + }, + dev: { + alias: 'd', + description: 'Build Docker dev images (with -dev suffix)', + default: false, + type: 'boolean', + }, + }, + (args) => { + logger.log(`Running command '${args._[0]} ${args._[1]}' with arguments ${longArguments(args)}`); + runCliCommand(() => { + buildDockerImages(args.npmRegistry, args.npmTag, args.dockerTag, args.dev); + }); + } + ) + .command( + 'publish', + 'Publish Docker images', + { + 'docker-tag': { + alias: 'g', + description: 'Docker tag to build the images under', + default: 'latest', + type: 'string', + }, + dev: { + alias: 'd', + description: 'Build Docker dev images (with -dev suffix)', + default: false, + type: 'boolean', + }, + }, + (args) => { + logger.log(`Running command '${args._[0]} ${args._[1]}' with arguments ${longArguments(args)}`); + runCliCommand(() => { + publishDockerImages(args.dockerTag, args.dev); + }); + } + ) + .help() + .demandCommand(1) + .strict(); + }) .help() .demandCommand(1) .strict() diff --git a/docker/scripts/build-docker-images.ts b/docker/scripts/docker.ts similarity index 65% rename from docker/scripts/build-docker-images.ts rename to docker/scripts/docker.ts index a6c4d9b4ba..2e9c4f6cca 100644 --- a/docker/scripts/build-docker-images.ts +++ b/docker/scripts/docker.ts @@ -26,3 +26,23 @@ export const buildDockerImages = (npmRegistry: string, npmTag: string, dockerTag `docker build --no-cache --build-arg npmRegistryUrl=${npmRegistryUrl} --build-arg npmTag=${npmTag} --tag api3/airnode-client${devSuffix}:${dockerTag} --file /app/airnode-client/Dockerfile /app/airnode-client` ); }; + +const loginDockerHub = () => { + const username = process.env.DOCKERHUB_USERNAME; + const password = process.env.DOCKERHUB_TOKEN; + + if (!username || !password) { + throw new Error('Missing DockerHub credentials'); + } + + runCommand(`docker login --password-stdin --username ${username}`, { input: password }); +}; + +export const publishDockerImages = (dockerTag: string, dev: boolean) => { + const devSuffix = dev ? '-dev' : ''; + + loginDockerHub(); + runCommand(`docker push api3/airnode-admin${devSuffix}:${dockerTag}`); + runCommand(`docker push api3/airnode-deployer${devSuffix}:${dockerTag}`); + runCommand(`docker push api3/airnode-client${devSuffix}:${dockerTag}`); +}; diff --git a/docker/scripts/utils.ts b/docker/scripts/utils.ts index b78fe724ee..34eae2ed64 100644 --- a/docker/scripts/utils.ts +++ b/docker/scripts/utils.ts @@ -1,9 +1,13 @@ import { execSync, ExecSyncOptions } from 'child_process'; import { statSync, readFileSync } from 'fs'; +import omit from 'lodash/omit'; import { logger } from '@api3/airnode-utilities'; export const runCommand = (command: string, options?: ExecSyncOptions) => { - logger.log(`Running command: '${command}'${options ? ` with options ${JSON.stringify(options)}` : ''}`); + // Omitting `input` as it's used for passing the DockerHub password and we don't want to log it + logger.log( + `Running command: '${command}'${options ? ` with options ${JSON.stringify(omit(options, ['input']))}` : ''}` + ); try { return execSync(command, options)?.toString().trim(); } catch (e) { diff --git a/package.json b/package.json index 0834cb41b1..11ff90eb5a 100644 --- a/package.json +++ b/package.json @@ -45,15 +45,17 @@ "dev:invoke": "(cd packages/airnode-node && yarn run dev:invoke)", "dev:list": "(cd packages/airnode-operation && yarn run dev:list)", "dev:stop": "(cd packages/airnode-operation && yarn run dev:stop)", - "docker:build:images:latest": "yarn docker:scripts:build-docker-images:latest", - "docker:build:images:local": "yarn docker:scripts:npm-registry:start && yarn docker:scripts:publish-packages:local && yarn docker:scripts:build-docker-images:local && yarn docker:scripts:npm-registry:stop", + "docker:build:images:latest": "yarn docker:scripts:docker:build:latest", + "docker:build:images:local": "yarn docker:scripts:npm-registry:start && yarn docker:scripts:publish-packages:local && yarn docker:scripts:docker:build:local && yarn docker:scripts:npm-registry:stop", "docker:build:latest": "yarn docker:build:packaging && yarn docker:build:images:latest", "docker:build:local": "yarn docker:build:packaging && yarn docker:build:images:local", "docker:build:packaging": "yarn docker:scripts:build && docker build --tag api3/airnode-packaging:latest --file docker/Dockerfile .", "docker:scripts:build": "ncc build docker/scripts/cli.ts -o docker/scripts/dist --no-cache --minify --source-map --transpile-only", - "docker:scripts:build-docker-images": "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest build-docker-images", - "docker:scripts:build-docker-images:latest": "yarn docker:scripts:build-docker-images", - "docker:scripts:build-docker-images:local": "yarn docker:scripts:build-docker-images --npm-registry local --npm-tag local --docker-tag local", + "docker:scripts:docker:build": "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest docker build", + "docker:scripts:docker:build:latest": "yarn docker:scripts:docker:build", + "docker:scripts:docker:build:local": "yarn docker:scripts:docker:build --npm-registry local --npm-tag local --docker-tag local", + "docker:scripts:docker:publish": "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e DOCKERHUB_USERNAME -e DOCKERHUB_TOKEN api3/airnode-packaging:latest docker publish", + "docker:scripts:docker:publish:latest": "yarn docker:scripts:docker:publish", "docker:scripts:npm-registry": "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest npm-registry", "docker:scripts:npm-registry:start": "yarn docker:scripts:npm-registry start", "docker:scripts:npm-registry:stop": "yarn docker:scripts:npm-registry stop",