feat(k8s): introduce Kubernetes lease lock mechanism

src/commands/account.ts

@@ -14,18 +14,16 @@
  * limitations under the License.
  *
  */
-
 import chalk from 'chalk'
 import { BaseCommand } from './base.ts'
 import { SoloError, IllegalArgumentError } from '../core/errors.ts'
 import { flags } from './index.ts'
 import { Listr } from 'listr2'
 import * as prompts from './prompts.ts'
-import { constants } from '../core/index.ts'
+import { constants, type AccountManager } from '../core/index.ts'
 import { type AccountId, AccountInfo, HbarUnit, PrivateKey } from '@hashgraph/sdk'
 import { FREEZE_ADMIN_ACCOUNT } from '../core/constants.ts'
-import { type AccountManager } from '../core/account_manager.ts'
-import { type Opts } from '../types/index.ts'
+import type { Opts } from '../types/index.ts'
 
 export class AccountCommand extends BaseCommand {
   private readonly accountManager: AccountManager
@@ -249,6 +247,7 @@ export class AccountCommand extends BaseCommand {
 
   async create (argv: any) {
     const self = this
+    const lease = self.leaseManager.instantiateLease()
 
     interface Context {
       config: {
@@ -292,6 +291,8 @@ export class AccountCommand extends BaseCommand {
           self.logger.debug('Initialized config', { config })
 
           await self.accountManager.loadNodeClient(ctx.config.namespace)
+
+          return lease.buildAcquireTask(task)
         }
       },
       {
@@ -314,6 +315,7 @@ export class AccountCommand extends BaseCommand {
     } catch (e: Error | any) {
       throw new SoloError(`Error in creating account: ${e.message}`, e)
     } finally {
+      await lease.release()
       await this.closeConnections()
     }
 

src/commands/base.ts

@@ -18,17 +18,17 @@
 import paths from 'path'
 import { MissingArgumentError } from '../core/errors.ts'
 import { ShellRunner } from '../core/shell_runner.ts'
-import { type ChartManager, type ConfigManager, type Helm, type K8 } from '../core/index.ts'
-import { type DependencyManager } from '../core/dependency_managers/index.ts'
-import { type CommandFlag, type Opts } from '../types/index.ts'
+import type {  ChartManager,  ConfigManager,  Helm,  K8,  DependencyManager, LeaseManager } from '../core/index.ts'
+import type {  CommandFlag,  Opts } from '../types/index.ts'
 
 export class BaseCommand extends ShellRunner {
   protected readonly helm: Helm
   protected readonly k8: K8
   protected readonly chartManager: ChartManager
   protected readonly configManager: ConfigManager
   protected readonly depManager: DependencyManager
-  protected readonly _configMaps: Map<string, any>
+  protected readonly leaseManager: LeaseManager
+  protected readonly _configMaps = new Map<string, any>()
 
   constructor (opts: Opts) {
     if (!opts || !opts.logger) throw new Error('An instance of core/SoloLogger is required')
@@ -45,7 +45,7 @@ export class BaseCommand extends ShellRunner {
     this.chartManager = opts.chartManager
     this.configManager = opts.configManager
     this.depManager = opts.depManager
-    this._configMaps = new Map()
+    this.leaseManager = opts.leaseManager
   }
 
   async prepareChartPath (chartDir: string, chartRepo: string, chartReleaseName: string) {

src/commands/cluster.ts

@@ -85,7 +85,6 @@ export class ClusterCommand extends BaseCommand {
             flags.deployPrometheusStack
           ])
 
-          // prepare config
           ctx.config = {
             chartDir: self.configManager.getFlag<string>(flags.chartDirectory) as string,
             clusterSetupNamespace: self.configManager.getFlag<string>(flags.clusterSetupNamespace) as string,
@@ -160,6 +159,7 @@ export class ClusterCommand extends BaseCommand {
 
   async reset (argv: any) {
     const self = this
+    const lease = self.leaseManager.instantiateLease()
 
     interface Context {
       config: {
@@ -195,6 +195,8 @@ export class ClusterCommand extends BaseCommand {
           if (!ctx.isChartInstalled) {
             throw new SoloError('No chart found for the cluster')
           }
+
+          return lease.buildAcquireTask(task)
         }
       },
       {
@@ -217,6 +219,8 @@ export class ClusterCommand extends BaseCommand {
       await tasks.run()
     } catch (e: Error | any) {
       throw new SoloError('Error on cluster reset', e)
+    } finally {
+      await lease.release()
     }
 
     return true

src/commands/flags.ts

@@ -18,7 +18,7 @@ import { constants } from '../core/index.ts'
 import * as core from '../core/index.ts'
 import * as version from '../../version.ts'
 import path from 'path'
-import { type CommandFlag } from '../types/index.ts'
+import type { CommandFlag } from '../types/index.ts'
 
 /**
  * Set flag from the flag option

src/commands/mirror_node.ts

@@ -17,12 +17,11 @@
 import { ListrEnquirerPromptAdapter } from '@listr2/prompt-adapter-enquirer'
 import { Listr } from 'listr2'
 import { SoloError, IllegalArgumentError, MissingArgumentError } from '../core/errors.ts'
-import { constants, type ProfileManager } from '../core/index.ts'
+import { constants, type ProfileManager, type AccountManager } from '../core/index.ts'
 import { BaseCommand } from './base.ts'
 import * as flags from './flags.ts'
 import * as prompts from './prompts.ts'
 import { getFileContents, getEnvValue } from '../core/helpers.ts'
-import { type AccountManager } from '../core/account_manager.ts'
 import { type PodName } from '../types/aliases.ts'
 import { type Opts } from '../types/index.ts'
 
@@ -61,13 +60,6 @@ export class MirrorNodeCommand extends BaseCommand {
     ]
   }
 
-  /**
-   * @param tlsClusterIssuerType
-   * @param enableHederaExplorerTls
-   * @param namespace
-   * @param hederaExplorerTlsLoadBalancerIp
-   * @param hederaExplorerTlsHostName
-   */
   getTlsValueArguments (tlsClusterIssuerType: string, enableHederaExplorerTls: boolean, namespace: string,
     hederaExplorerTlsLoadBalancerIp: string, hederaExplorerTlsHostName: string) {
     let valuesArg = ''
@@ -126,6 +118,7 @@ export class MirrorNodeCommand extends BaseCommand {
 
   async deploy (argv: any) {
     const self = this
+    const lease = self.leaseManager.instantiateLease()
 
     interface MirrorNodeDeployConfigClass {
       chartDirectory: string
@@ -184,6 +177,8 @@ export class MirrorNodeCommand extends BaseCommand {
           }
 
           await self.accountManager.loadNodeClient(ctx.config.namespace)
+
+          return lease.buildAcquireTask(task)
         }
       },
       {
@@ -327,6 +322,7 @@ export class MirrorNodeCommand extends BaseCommand {
     } catch (e: Error | any) {
       throw new SoloError(`Error starting node: ${e.message}`, e)
     } finally {
+      await lease.release()
       await self.accountManager.close()
     }
 
@@ -335,6 +331,7 @@ export class MirrorNodeCommand extends BaseCommand {
 
   async destroy (argv: any) {
     const self = this
+    const lease = self.leaseManager.instantiateLease()
 
     interface Context {
       config: {
@@ -384,6 +381,8 @@ export class MirrorNodeCommand extends BaseCommand {
           }
 
           await self.accountManager.loadNodeClient(ctx.config.namespace)
+
+          return lease.buildAcquireTask(task)
         }
       },
       {
@@ -425,6 +424,7 @@ export class MirrorNodeCommand extends BaseCommand {
     } catch (e: Error | any) {
       throw new SoloError(`Error starting node: ${e.message}`, e)
     } finally {
+      await lease.release()
       await self.accountManager.close()
     }
 

src/commands/network.ts

@@ -103,7 +103,8 @@
 
   async prepareValuesArg (config: {chartDirectory?: string; app?: string; nodeAliases?: string[]; debugNodeAlias?: NodeAlias;
     enablePrometheusSvcMonitor?: boolean; releaseTag?: string; persistentVolumeClaims?: string;
-    valuesFile?: string; } = {}) {
+    valuesFile?: string; } = {}
+  ) {
     let valuesArg = config.chartDirectory ? `-f ${path.join(config.chartDirectory, 'solo-deployment', 'values.yaml')}` : ''
 
     if (config.app !== constants.HEDERA_APP_NAME) {
@@ -216,6 +217,7 @@
   /** Run helm install and deploy network components */
   async deploy (argv: any) {
     const self = this
+    const lease = self.leaseManager.instantiateLease()
 
     interface Context {
       config: NetworkDeployConfigClass
@@ -226,6 +228,7 @@
         title: 'Initialize',
         task: async (ctx, task) => {
           ctx.config = await self.prepareConfig(task, argv)
+          return lease.buildAcquireTask(task)
         }
       },
       {
@@ -383,13 +386,16 @@
       await tasks.run()
     } catch (e: Error | any) {
       throw new SoloError(`Error installing chart ${constants.SOLO_DEPLOYMENT_CHART}`, e)
+    } finally {
+      await lease.release()
     }
 
     return true
   }
 
   async destroy (argv: any) {
     const self = this
+    const lease = self.leaseManager.instantiateLease()
 
     interface Context {
       config: {
@@ -427,6 +433,8 @@
             deleteSecrets: self.configManager.getFlag<boolean>(flags.deleteSecrets) as boolean,
             namespace: self.configManager.getFlag<string>(flags.namespace) as string
           }
+
+          return lease.buildAcquireTask(task)
         }
       },
       {
@@ -470,6 +478,8 @@
       await tasks.run()
     } catch (e: Error | any) {
       throw new SoloError('Error destroying network', e)
+    } finally {
+      await lease.release()
     }
 
     return true
@@ -478,6 +488,7 @@
   /** Run helm upgrade to refresh network components with new settings */
   async refresh (argv: any) {
     const self = this
+    const lease = self.leaseManager.instantiateLease()
 
     interface Context {
       config: NetworkDeployConfigClass
@@ -488,6 +499,7 @@
         title: 'Initialize',
         task: async (ctx, task) => {
           ctx.config = await self.prepareConfig(task, argv)
+          return lease.buildAcquireTask(task)
         }
       },
       {
@@ -520,6 +532,8 @@
       await tasks.run()
     } catch (e: Error | any) {
       throw new SoloError(`Error upgrading chart ${constants.SOLO_DEPLOYMENT_CHART}`, e)
+    } finally {
+      await lease.release()
     }
 
     return true

src/commands/node/configs.ts

@@ -23,8 +23,8 @@ import path from 'path'
 import fs from 'fs'
 import { validatePath } from '../../core/helpers.ts'
 import * as flags from '../flags.ts'
-import { type NodeAlias, type NodeAliases, type PodName } from '../../types/aliases.js'
-import { type NetworkNodeServices } from '../../core/network_node_services.js'
+import { type NodeAlias, type NodeAliases, type PodName } from '../../types/aliases.ts'
+import { type NetworkNodeServices } from '../../core/network_node_services.ts'
 
 export const PREPARE_UPGRADE_CONFIGS_NAME = 'prepareUpgradeConfig'
 export const DOWNLOAD_GENERATED_FILES_CONFIGS_NAME = 'downloadGeneratedFilesConfig'