Deployer missing files (#1599)

* Add check for missing airnode deployment files

* Enable ora warn and info logging while spinner is spinning

* Add changeset

* Fix missing latestDeployment children object

* Skip fetching deployment files if config.json or secrets.env is missing

* Refactor checking for bucket missing files, add check to deployAirnode

* Fix missing file check

* Fix missing files object keys

* Remove use of TF_STATE_FILENAME to fix tests

* Remove duplicate logging of invalid bucket directory

* Wrap file fetching and JSON.parse with go, refactor names

.changeset/mighty-dryers-cheat.md

@@ -0,0 +1,5 @@
+---
+'@api3/airnode-deployer': patch
+---
+
+Add check for missing secrets.env, config.json, default.tfstate files in bucket

packages/airnode-deployer/src/infrastructure/index.ts

@@ -36,6 +36,7 @@ import {
   FileSystemType,
   deploymentComparator,
   Bucket,
+  getMissingBucketFiles,
 } from '../utils/infrastructure';
 import { version as nodeVersion } from '../../package.json';
 import { deriveAirnodeAddress } from '../utils';
@@ -333,14 +334,35 @@ export const deployAirnode = async (config: Config, configPath: string, secretsP
     const stageDirectory = getStageDirectory(directoryStructure, airnodeAddress, stage);
     if (stageDirectory) {
       logger.debug(`Deployment '${bucketStagePath}' already exists`);
+
+      const bucketMissingFiles = getMissingBucketFiles(directoryStructure);
+      if (
+        bucketMissingFiles[airnodeAddress] &&
+        bucketMissingFiles[airnodeAddress][stage] &&
+        bucketMissingFiles[airnodeAddress][stage].length !== 0
+      ) {
+        throw new Error(
+          `Can't update an Airnode with missing files: ${bucketMissingFiles[airnodeAddress][stage].join(
+            ', '
+          )}. Deployer commands may fail and manual removal may be necessary.`
+        );
+      }
+
       const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0];
       const bucketConfigPath = `${bucketStagePath}/${latestDeployment}/config.json`;
       logger.debug(`Fetching configuration file '${bucketConfigPath}'`);
-      const remoteConfig = JSON.parse(
-        await cloudProviderLib[type].getFileFromBucket(bucket.name, bucketConfigPath)
-      ) as Config;
+      const goGetRemoteConfigFileFromBucket = await go(() =>
+        cloudProviderLib[type].getFileFromBucket(bucket!.name, bucketConfigPath)
+      );
+      if (!goGetRemoteConfigFileFromBucket.success) {
+        throw new Error(`Failed to fetch configuration file. Error: ${goGetRemoteConfigFileFromBucket.error.message}`);
+      }
+      const goRemoteConfig = goSync(() => JSON.parse(goGetRemoteConfigFileFromBucket.data));
+      if (!goRemoteConfig.success) {
+        throw new Error(`Failed to parse configuration file. Error: ${goRemoteConfig.error.message}`);
+      }
 
-      const remoteNodeSettings = remoteConfig.nodeSettings;
+      const remoteNodeSettings = goRemoteConfig.data.nodeSettings;
       const remoteCloudProvider = remoteNodeSettings.cloudProvider as CloudProvider;
       if (remoteNodeSettings.nodeVersion !== nodeVersion) {
         throw new Error(
@@ -440,6 +462,8 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy
   }
 
   const directoryStructure = await cloudProviderLib[cloudProviderType].getBucketDirectoryStructure(bucket.name);
+  const bucketMissingFiles = getMissingBucketFiles(directoryStructure);
+
   for (const [airnodeAddress, addressDirectory] of Object.entries(directoryStructure)) {
     if (addressDirectory.type !== FileSystemType.Directory) {
       logger.warn(
@@ -461,19 +485,43 @@ async function fetchDeployments(cloudProviderType: CloudProvider['type'], deploy
       }
 
       const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0];
+
+      if (
+        bucketMissingFiles[airnodeAddress] &&
+        bucketMissingFiles[airnodeAddress][stage] &&
+        bucketMissingFiles[airnodeAddress][stage].length !== 0
+      ) {
+        continue;
+      }
+
       const bucketLatestDeploymentPath = `${airnodeAddress}/${stage}/${latestDeployment}`;
 
       const bucketConfigPath = `${bucketLatestDeploymentPath}/config.json`;
       logger.debug(`Fetching configuration file '${bucketConfigPath}'`);
-      const config = JSON.parse(
-        await cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketConfigPath)
+      const goGetConfigFileFromBucket = await go(() =>
+        cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketConfigPath)
       );
+      if (!goGetConfigFileFromBucket.success) {
+        logger.warn(`Failed to fetch configuration file. Error: ${goGetConfigFileFromBucket.error.message} Skipping.`);
+        continue;
+      }
+      const goConfig = goSync(() => JSON.parse(goGetConfigFileFromBucket.data));
+      if (!goConfig.success) {
+        logger.warn(`Failed to parse configuration file. Error: ${goConfig.error.message} Skipping.`);
+        continue;
+      }
+
+      logger.debug(`Fetching secrets file '${bucketConfigPath}'`);
       const bucketSecretsPath = `${bucketLatestDeploymentPath}/secrets.env`;
-      logger.debug(`Fetching secrets file '${bucketSecretsPath}'`);
-      const secrets = dotenv.parse(
-        await cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketSecretsPath)
+      const goGetSecretsFileFromBucket = await go(() =>
+        cloudProviderLib[cloudProviderType].getFileFromBucket(bucket.name, bucketSecretsPath)
       );
-      const interpolatedConfig = unsafeParseConfigWithSecrets(config, secrets);
+      if (!goGetSecretsFileFromBucket.success) {
+        logger.warn(`Failed to fetch secrets file. Error: ${goGetSecretsFileFromBucket.error.message} Skipping.`);
+        continue;
+      }
+      const secrets = dotenv.parse(goGetSecretsFileFromBucket.data);
+      const interpolatedConfig = unsafeParseConfigWithSecrets(goConfig.data, secrets);
 
       const cloudProvider = interpolatedConfig.nodeSettings.cloudProvider as CloudProvider;
       const airnodeVersion = interpolatedConfig.nodeSettings.nodeVersion;

packages/airnode-deployer/src/utils/infrastructure.ts

@@ -1,11 +1,15 @@
 import { randomBytes } from 'crypto';
 import isArray from 'lodash/isArray';
+import isEmpty from 'lodash/isEmpty';
+import difference from 'lodash/difference';
 import { compareVersions } from 'compare-versions';
 import * as logger from './logger';
 import { Deployment } from '../infrastructure';
 
 type CommandArg = string | [string, string] | [string, string, string];
 
+const DEPLOYMENT_REQUIRED_FILE_NAMES = ['config.json', 'secrets.env', 'default.tfstate'];
+
 export function formatTerraformArguments(args: CommandArg[]) {
   return args
     .map((arg) => {
@@ -163,3 +167,38 @@ export const deploymentComparator = (a: Deployment, b: Deployment) => {
 
   return compareVersions(a.airnodeVersion, b.airnodeVersion);
 };
+
+export const getMissingBucketFiles = (
+  directoryStructure: DirectoryStructure
+): Record<string, Record<string, string[]>> =>
+  Object.entries(directoryStructure).reduce((acc, [airnodeAddress, addressDirectory]) => {
+    if (addressDirectory.type !== FileSystemType.Directory) {
+      return acc;
+    }
+
+    const checkedAddressDirectory = Object.entries(addressDirectory.children).reduce((acc, [stage, stageDirectory]) => {
+      if (stageDirectory.type !== FileSystemType.Directory) {
+        return acc;
+      }
+
+      const latestDeployment = Object.keys(stageDirectory.children).sort().reverse()[0];
+      const latestDepolymentFileNames = Object.keys(
+        (stageDirectory.children[latestDeployment] as Directory)?.children || {}
+      );
+
+      const missingRequiredFiles = difference(DEPLOYMENT_REQUIRED_FILE_NAMES, latestDepolymentFileNames);
+      if (isEmpty(missingRequiredFiles)) {
+        return { ...acc, [airnodeAddress]: { [stage]: [] } };
+      }
+
+      logger.warn(
+        `Airnode '${airnodeAddress}' with stage '${stage}' is missing files: ${missingRequiredFiles.join(
+          ', '
+        )}. Deployer commands may fail and manual removal may be necessary.`
+      );
+
+      return { ...acc, [airnodeAddress]: { [stage]: missingRequiredFiles } };
+    }, {});
+
+    return { ...acc, ...checkedAddressDirectory };
+  }, {});

packages/airnode-deployer/src/utils/logger.ts

@@ -36,15 +36,25 @@ export function fail(text: string) {
 }
 
 export function warn(text: string) {
+  const currentOra = getSpinner();
+  if (currentOra.isSpinning) {
+    currentOra.clear();
+    currentOra.frame();
+  }
   oraInstance().warn(text);
 }
 
 export function info(text: string) {
+  const currentOra = getSpinner();
+  if (currentOra.isSpinning) {
+    currentOra.clear();
+    currentOra.frame();
+  }
   oraInstance().info(text);
 }
 
 export function debug(text: string) {
-  if (debugModeFlag) oraInstance().info(text);
+  if (debugModeFlag) info(text);
 }
 
 export function debugSpinner(text: string) {