Switch to balena-compose

Removes a bunch of individual dependencies by switching to `@balena/compose` which (currently) groups and manages those dependencies together in one package.

Change-type: minor

lib/commands/deploy.ts

@@ -16,7 +16,7 @@
  */
 
 import { flags } from '@oclif/command';
-import type { ImageDescriptor } from 'resin-compose-parse';
+import type { ImageDescriptor } from '@balena/compose/dist/parse';
 
 import Command from '../command';
 import { ExpectedError } from '../errors';

lib/commands/push.ts

@@ -22,7 +22,7 @@ import { getBalenaSdk, stripIndent } from '../utils/lazy';
 import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
 import type { BalenaSDK } from 'balena-sdk';
 import { ExpectedError, instanceOf } from '../errors';
-import { RegistrySecrets } from 'resin-multibuild';
+import { RegistrySecrets } from '@balena/compose/dist/multibuild';
 import { lowercaseIfSlug } from '../utils/normalization';
 import {
 	applyReleaseTagKeysAndValues,

lib/utils/compose-types.d.ts

@@ -15,8 +15,11 @@
  * limitations under the License.
  */
 
-import type { ImageModel, ReleaseModel } from 'balena-release/build/models';
-import type { Composition, ImageDescriptor } from 'resin-compose-parse';
+import type {
+	ImageModel,
+	ReleaseModel,
+} from '@balena/compose/dist/release/models';
+import type { Composition, ImageDescriptor } from '@balena/compose/dist/parse';
 import type { Pack } from 'tar-stream';
 
 interface Image {
@@ -39,7 +42,7 @@ export interface BuiltImage {
 
 export interface TaggedImage {
 	localImage: import('dockerode').Image;
-	serviceImage: import('balena-release/build/models').ImageModel;
+	serviceImage: import('@balena/compose/dist/release/models').ImageModel;
 	serviceName: string;
 	logs: string;
 	props: BuiltImage.props;
@@ -78,7 +81,9 @@ export interface ComposeProject {
 }
 
 export interface Release {
-	client: ReturnType<typeof import('balena-release').createClient>;
+	client: ReturnType<
+		typeof import('@balena/compose/dist/release').createClient
+	>;
 	release: Pick<
 		ReleaseModel,
 		| 'id'

lib/utils/compose.ts

@@ -19,7 +19,7 @@ import type { Renderer } from './compose_ts';
 import type * as SDK from 'balena-sdk';
 import type Dockerode = require('dockerode');
 import * as path from 'path';
-import type { Composition, ImageDescriptor } from 'resin-compose-parse';
+import type { Composition, ImageDescriptor } from '@balena/compose/dist/parse';
 import type {
 	BuiltImage,
 	ComposeOpts,
@@ -64,7 +64,7 @@ export function createProject(
 ): ComposeProject {
 	const yml = require('js-yaml') as typeof import('js-yaml');
 	const compose =
-		require('resin-compose-parse') as typeof import('resin-compose-parse');
+		require('@balena/compose/dist/parse') as typeof import('@balena/compose/dist/parse');
 
 	// both methods below may throw.
 	const rawComposition = yml.load(composeStr);
@@ -107,7 +107,7 @@ export const createRelease = async function (
 	const _ = require('lodash') as typeof import('lodash');
 	const crypto = require('crypto') as typeof import('crypto');
 	const releaseMod =
-		require('balena-release') as typeof import('balena-release');
+		require('@balena/compose/dist/release') as typeof import('@balena/compose/dist/release');
 
 	const client = releaseMod.createClient({ apiEndpoint, auth });
 
@@ -214,7 +214,7 @@ export const getPreviousRepos = (
 					image: [SDK.Image];
 				}>;
 				const { getRegistryAndName } =
-					require('resin-multibuild') as typeof import('resin-multibuild');
+					require('@balena/compose/dist/multibuild') as typeof import('@balena/compose/dist/multibuild');
 				return Promise.all(
 					images.map(function (d) {
 						const imageName = d.image[0].is_stored_at__image_location || '';

lib/utils/compose_ts.ts

@@ -16,7 +16,7 @@
  */
 import { flags } from '@oclif/command';
 import { BalenaSDK } from 'balena-sdk';
-import type { TransposeOptions } from 'docker-qemu-transpose';
+import type { TransposeOptions } from '@balena/compose/dist/emulate';
 import type * as Dockerode from 'dockerode';
 import { promises as fs } from 'fs';
 import jsyaml = require('js-yaml');
@@ -26,8 +26,8 @@ import type {
 	BuildConfig,
 	Composition,
 	ImageDescriptor,
-} from 'resin-compose-parse';
-import type * as MultiBuild from 'resin-multibuild';
+} from '@balena/compose/dist/parse';
+import type * as MultiBuild from '@balena/compose/dist/multibuild';
 import * as semver from 'semver';
 import type { Duplex, Readable } from 'stream';
 import type { Pack } from 'tar-stream';
@@ -118,7 +118,7 @@ export async function loadProject(
 	image?: string,
 	imageTag?: string,
 ): Promise<ComposeProject> {
-	const compose = await import('resin-compose-parse');
+	const compose = await import('@balena/compose/dist/parse');
 	const { createProject } = await import('./compose');
 	let composeName: string;
 	let composeStr: string;
@@ -262,7 +262,7 @@ export async function buildProject(
 	opts: BuildProjectOpts,
 ): Promise<BuiltImage[]> {
 	await checkBuildSecretsRequirements(opts.docker, opts.projectPath);
-	const compose = await import('resin-compose-parse');
+	const compose = await import('@balena/compose/dist/parse');
 	const imageDescriptors = compose.parse(opts.composition);
 	const renderer = await startRenderer({ imageDescriptors, ...opts });
 	let buildSummaryByService: Dictionary<string> | undefined;
@@ -333,7 +333,7 @@ async function $buildProject(
 	logger.logDebug('Prepared tasks; building...');
 
 	const { BALENA_ENGINE_TMP_PATH } = await import('../config');
-	const builder = await import('resin-multibuild');
+	const builder = await import('@balena/compose/dist/multibuild');
 
 	const builtImages = await builder.performBuilds(
 		tasks,
@@ -481,8 +481,9 @@ async function qemuTransposeBuildStream({
 		throw new Error(`No buildStream for task '${task.tag}'`);
 	}
 
-	const transpose = await import('docker-qemu-transpose');
-	const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
+	const transpose = await import('@balena/compose/dist/emulate');
+	const { toPosixPath } = (await import('@balena/compose/dist/multibuild'))
+		.PathUtils;
 
 	const transposeOptions: TransposeOptions = {
 		hostQemuPath: toPosixPath(binPath),
@@ -508,9 +509,9 @@ async function setTaskProgressHooks({
 	inlineLogs?: boolean;
 	renderer: Renderer;
 	task: BuildTaskPlus;
-	transposeOptions?: import('docker-qemu-transpose').TransposeOptions;
+	transposeOptions?: import('@balena/compose/dist/emulate').TransposeOptions;
 }) {
-	const transpose = await import('docker-qemu-transpose');
+	const transpose = await import('@balena/compose/dist/emulate');
 	// Get the service-specific log stream
 	const logStream = renderer.streams[task.serviceName];
 	task.logBuffer = [];
@@ -724,16 +725,16 @@ export async function getServiceDirsFromComposition(
  *
  * The `image` argument may therefore refer to either a `build` or `image` property
  * of a service in a docker-compose.yml file, which is a bit confusing but it matches
- * the `ImageDescriptor.image` property as defined by `resin-compose-parse`.
+ * the `ImageDescriptor.image` property as defined by `@balena/compose/parse`.
  *
- * Note that `resin-compose-parse` "normalizes" the docker-compose.yml file such
+ * Note that `@balena/compose/parse` "normalizes" the docker-compose.yml file such
  * that, if `services.service.build` is a string, it is converted to a BuildConfig
  * object with the string value assigned to `services.service.build.context`:
- * https://github.com/balena-io-modules/resin-compose-parse/blob/v2.1.3/src/compose.ts#L166-L167
+ * https://github.com/balena-io-modules/balena-compose/blob/v0.1.0/lib/parse/compose.ts#L166-L167
  * This is why this implementation works when `services.service.build` is defined
  * as a string in the docker-compose.yml file.
  *
- * @param image The `ImageDescriptor.image` attribute parsed with `resin-compose-parse`
+ * @param image The `ImageDescriptor.image` attribute parsed with `@balena/compose/parse`
  */
 export function isBuildConfig(
 	image: string | BuildConfig,
@@ -759,7 +760,8 @@ export async function tarDirectory(
 	}: TarDirectoryOptions,
 ): Promise<import('stream').Readable> {
 	const { filterFilesWithDockerignore } = await import('./ignore');
-	const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
+	const { toPosixPath } = (await import('@balena/compose/dist/multibuild'))
+		.PathUtils;
 
 	let readFile: (file: string) => Promise<Buffer>;
 	if (process.platform === 'win32') {
@@ -941,7 +943,7 @@ async function parseRegistrySecrets(
 			throw new ExpectedError('Filename must end with .json, .yml or .yaml');
 		}
 		const raw = (await fs.readFile(secretsFilename)).toString();
-		const multiBuild = await import('resin-multibuild');
+		const multiBuild = await import('@balena/compose/dist/multibuild');
 		const registrySecrets =
 			new multiBuild.RegistrySecretValidator().validateRegistrySecrets(
 				isYaml ? require('js-yaml').load(raw) : JSON.parse(raw),
@@ -970,7 +972,7 @@ export async function makeBuildTasks(
 	releaseHash: string = 'unavailable',
 	preprocessHook?: (dockerfile: string) => string,
 ): Promise<MultiBuild.BuildTask[]> {
-	const multiBuild = await import('resin-multibuild');
+	const multiBuild = await import('@balena/compose/dist/multibuild');
 	const buildTasks = await multiBuild.splitBuildStream(composition, tarStream);
 
 	logger.logDebug('Found build tasks:');
@@ -1016,7 +1018,7 @@ async function performResolution(
 	releaseHash: string,
 	preprocessHook?: (dockerfile: string) => string,
 ): Promise<MultiBuild.BuildTask[]> {
-	const multiBuild = await import('resin-multibuild');
+	const multiBuild = await import('@balena/compose/dist/multibuild');
 	const resolveListeners: MultiBuild.ResolveListeners = {};
 	const resolvePromise = new Promise<never>((_resolve, reject) => {
 		resolveListeners.error = [reject];
@@ -1081,7 +1083,7 @@ async function validateSpecifiedDockerfile(
 	dockerfilePath: string,
 ): Promise<string> {
 	const { contains, toNativePath, toPosixPath } = (
-		await import('resin-multibuild')
+		await import('@balena/compose/dist/multibuild')
 	).PathUtils;
 
 	const nativeProjectPath = path.normalize(projectPath);
@@ -1241,7 +1243,7 @@ async function pushAndUpdateServiceImages(
 	token: string,
 	images: TaggedImage[],
 	afterEach: (
-		serviceImage: import('balena-release/build/models').ImageModel,
+		serviceImage: import('@balena/compose/dist/release/models').ImageModel,
 		props: object,
 	) => void,
 ) {
@@ -1326,12 +1328,14 @@ async function pushAndUpdateServiceImages(
 async function pushServiceImages(
 	docker: Dockerode,
 	logger: Logger,
-	pineClient: ReturnType<typeof import('balena-release').createClient>,
+	pineClient: ReturnType<
+		typeof import('@balena/compose/dist/release').createClient
+	>,
 	taggedImages: TaggedImage[],
 	token: string,
 	skipLogUpload: boolean,
 ): Promise<void> {
-	const releaseMod = await import('balena-release');
+	const releaseMod = await import('@balena/compose/dist/release');
 	logger.logInfo('Pushing images to registry...');
 	await pushAndUpdateServiceImages(
 		docker,
@@ -1361,8 +1365,8 @@ export async function deployProject(
 	skipLogUpload: boolean,
 	projectPath: string,
 	isDraft: boolean,
-): Promise<import('balena-release/build/models').ReleaseModel> {
-	const releaseMod = await import('balena-release');
+): Promise<import('@balena/compose/dist/release/models').ReleaseModel> {
+	const releaseMod = await import('@balena/compose/dist/release');
 	const { createRelease, tagServiceImages } = await import('./compose');
 	const tty = (await import('./tty'))(process.stdout);
 

lib/utils/device/deploy.ts

@@ -18,13 +18,13 @@
 import * as semver from 'balena-semver';
 import * as Docker from 'dockerode';
 import * as _ from 'lodash';
-import { Composition } from 'resin-compose-parse';
+import { Composition } from '@balena/compose/dist/parse';
 import {
 	BuildTask,
 	getAuthConfigObj,
 	LocalImage,
 	RegistrySecrets,
-} from 'resin-multibuild';
+} from '@balena/compose/dist/multibuild';
 import type { Readable } from 'stream';
 
 import { BALENA_ENGINE_TMP_PATH } from '../../config';
@@ -321,7 +321,7 @@ async function performBuilds(
 	opts: DeviceDeployOptions,
 	buildLogs?: Dictionary<string>,
 ): Promise<BuildTask[]> {
-	const multibuild = await import('resin-multibuild');
+	const multibuild = await import('@balena/compose/dist/multibuild');
 
 	const buildTasks = await makeBuildTasks(
 		composition,
@@ -370,7 +370,7 @@ async function performBuilds(
 	const imagesToRemove: string[] = [];
 
 	// Now tag any external images with the correct name that they should be,
-	// as this won't be done by resin-multibuild
+	// as this won't be done by @balena/compose/multibuild
 	await Promise.all(
 		localImages.map(async (localImage) => {
 			if (localImage.external) {
@@ -414,7 +414,7 @@ export async function rebuildSingleTask(
 	// this should provide the following callback
 	containerIdCb?: (id: string) => void,
 ): Promise<string> {
-	const multibuild = await import('resin-multibuild');
+	const multibuild = await import('@balena/compose/dist/multibuild');
 	// First we run the build task, to get the new image id
 	let buildLogs = '';
 	const logHandler = (_s: string, line: string) => {
@@ -533,10 +533,17 @@ async function assignDockerBuildOpts(
 	await Promise.all(
 		buildTasks.map(async (task: BuildTask) => {
 			task.dockerOpts = {
-				cachefrom: images,
-				labels: {
-					'io.resin.local.image': '1',
-					'io.resin.local.service': task.serviceName,
+				...(task.dockerOpts || {}),
+				...{
+					cachefrom: images,
+					labels: {
+						'io.resin.local.image': '1',
+						'io.resin.local.service': task.serviceName,
+					},
+					t: getImageNameFromTask(task),
+					nocache: opts.nocache,
+					forcerm: true,
+					pull: opts.pull,
 				},
 				t: getImageNameFromTask(task),
 				nocache: opts.nocache,

lib/utils/device/live.ts

@@ -21,8 +21,8 @@ import * as fs from 'fs';
 import Livepush, { ContainerNotRunningError } from 'livepush';
 import * as _ from 'lodash';
 import * as path from 'path';
-import type { Composition } from 'resin-compose-parse';
-import type { BuildTask } from 'resin-multibuild';
+import type { Composition } from '@balena/compose/dist/parse';
+import type { BuildTask } from '@balena/compose/dist/multibuild';
 
 import { instanceOf } from '../../errors';
 import Logger = require('../logger');

lib/utils/docker.ts

@@ -105,7 +105,7 @@ export interface BuildOpts {
 	cachefrom?: string[];
 	nocache?: boolean;
 	pull?: boolean;
-	registryconfig?: import('resin-multibuild').RegistrySecrets;
+	registryconfig?: import('@balena/compose/dist/multibuild').RegistrySecrets;
 	squash?: boolean;
 	t?: string; // only the tag portion of the image name, e.g. 'abc' in 'myimg:abc'
 }
@@ -132,7 +132,7 @@ export function generateBuildOpts(options: {
 	'cache-from'?: string;
 	nocache: boolean;
 	pull?: boolean;
-	'registry-secrets'?: import('resin-multibuild').RegistrySecrets;
+	'registry-secrets'?: import('@balena/compose/dist/multibuild').RegistrySecrets;
 	squash: boolean;
 	tag?: string;
 }): BuildOpts {

lib/utils/remote-build.ts

@@ -17,7 +17,7 @@ import type { BalenaSDK } from 'balena-sdk';
 import * as JSONStream from 'JSONStream';
 import * as readline from 'readline';
 import * as request from 'request';
-import { RegistrySecrets } from 'resin-multibuild';
+import { RegistrySecrets } from '@balena/compose/dist/multibuild';
 import type * as Stream from 'stream';
 import streamToPromise = require('stream-to-promise');
 import type { Pack } from 'tar-stream';