From 19c4f0d3b2182f8d2e40f993ddd53989a997117f Mon Sep 17 00:00:00 2001 From: c-bordon Date: Tue, 30 Apr 2024 08:27:32 -0300 Subject: [PATCH 1/6] Replace black mini parameters --- .../modules/allocation/vagrant/provider.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/deployability/modules/allocation/vagrant/provider.py b/deployability/modules/allocation/vagrant/provider.py index 38f64678e1..007eee5d50 100644 --- a/deployability/modules/allocation/vagrant/provider.py +++ b/deployability/modules/allocation/vagrant/provider.py @@ -279,14 +279,14 @@ def __remote_host(arch: str, action: str, os: str = None, instance_dir: Path = N ssh_password = client.get_secret_value(SecretId='devops_macstadium_m1_jenkins_password')['SecretString'] ssh_user = client.get_secret_value(SecretId='devops_macstadium_m1_jenkins_user')['SecretString'] except Exception as e: - raise ValueError('Could not get macOS macStadium server IP: ' + str(e) + '.') + raise ValueError('Could not get macOS macStadium ARM server IP: ' + str(e) + '.') try: tn = Telnet(server_ip, server_port, timeout) conn_ok = True tn.close() except Exception as e: - raise ValueError('Could not connect to macOS macStadium server: ' + str(e) + '.') + raise ValueError('Could not connect to macOS macStadium ARM server: ' + str(e) + '.') remote_host_parameters['server_ip'] = server_ip remote_host_parameters['ssh_password'] = ssh_password @@ -300,37 +300,37 @@ def __remote_host(arch: str, action: str, os: str = None, instance_dir: Path = N prlctl_output = subprocess.Popen(f"sshpass -p {ssh_password} ssh -o 'StrictHostKeyChecking no' {ssh_user}@{server_ip} {cmd}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode('utf-8') data_list = json.loads(prlctl_output) except Exception as e: - raise ValueError('Could not get VMs running on macStadium server: ' + str(e) + '.') + raise ValueError('Could not get VMs running on macStadium ARM server: ' + str(e) + '.') uuid_count = 0 for item in data_list: if 'uuid' in item: uuid_count += 1 if uuid_count < 2: - logger.info(f"macStadium server has less than 2 VMs running, deploying in this host.") + logger.info(f"macStadium ARM server has less than 2 VMs running, deploying in this host.") return remote_host_parameters else: - raise ValueError(f"macStadium server is full capacity, use AWS provider.") + raise ValueError(f"macStadium ARM server is full capacity, use AWS provider.") else: return remote_host_parameters if arch == 'amd64': try: - server_ip = client.get_secret_value(SecretId='devops_black_mini_jenkins_ip')['SecretString'] - ssh_password = client.get_secret_value(SecretId='devops_black_mini_jenkins_password')['SecretString'] - ssh_user = client.get_secret_value(SecretId='devops_black_mini_jenkins_user')['SecretString'] + server_ip = client.get_secret_value(SecretId='devops_macstadium_intel_ip')['SecretString'] + ssh_password = client.get_secret_value(SecretId='devops_macstadium_intel_password')['SecretString'] + ssh_user = client.get_secret_value(SecretId='devops_macstadium_intel_user')['SecretString'] except Exception as e: - raise ValueError('Could not get macOS Black mini server IP: ' + str(e) + '.') + raise ValueError('Could not get macOS macStadium Intel server IP: ' + str(e) + '.') try: tn = Telnet(server_ip, server_port, timeout) conn_ok = True tn.close() except Exception as e: - raise ValueError('Could not connect to macOS Black mini server: ' + str(e) + '.') + raise ValueError('Could not connect to macOS macStadium Intel server: ' + str(e) + '.') remote_host_parameters['server_ip'] = server_ip remote_host_parameters['ssh_password'] = ssh_password remote_host_parameters['ssh_user'] = ssh_user - remote_host_parameters['host_provider'] = 'black_mini' + remote_host_parameters['host_provider'] = 'macstadium' if conn_ok: if action == 'create': @@ -345,10 +345,10 @@ def __remote_host(arch: str, action: str, os: str = None, instance_dir: Path = N raise ValueError('Could not get server load average: ' + str(e) + '.') if float(load_average) <= 10.0 and float(cpu_usage) <= 70.0 and float(memory_usage) <= 75.0: - logger.info(f"Using the black mini server to deploy.") + logger.info(f"Using the macStadium Intel server to deploy.") return remote_host_parameters else: - raise ValueError(f"Black mini server is under heavy load, use AWS provider.") + raise ValueError(f"macStadium Intel server is under heavy load, use AWS provider.") else: return remote_host_parameters From c1a494d931cc256cc66927bd9b101b4df28b7612 Mon Sep 17 00:00:00 2001 From: c-bordon Date: Thu, 2 May 2024 14:57:48 -0300 Subject: [PATCH 2/6] MacOS Intel machine deployments are updated --- .../modules/allocation/static/specs/os.yml | 6 +++++ ...t_black_mini.j2 => vagrant_Virtual_box.j2} | 4 ---- ...macStadium.j2 => vagrant_parallels_arm.j2} | 0 .../templates/vagrant_parallels_intel.j2 | 19 +++++++++++++++ .../vagrant/helpers/vagrant_script.sh | 23 +++++++++++++++++++ .../modules/allocation/vagrant/instance.py | 13 ++++------- .../modules/allocation/vagrant/provider.py | 12 +++++++--- 7 files changed, 62 insertions(+), 15 deletions(-) rename deployability/modules/allocation/static/templates/{vagrant_black_mini.j2 => vagrant_Virtual_box.j2} (71%) rename deployability/modules/allocation/static/templates/{vagrant_macStadium.j2 => vagrant_parallels_arm.j2} (100%) mode change 100755 => 100644 create mode 100755 deployability/modules/allocation/static/templates/vagrant_parallels_intel.j2 create mode 100644 deployability/modules/allocation/vagrant/helpers/vagrant_script.sh diff --git a/deployability/modules/allocation/static/specs/os.yml b/deployability/modules/allocation/static/specs/os.yml index 1e49ce4db3..05b36a2ce6 100644 --- a/deployability/modules/allocation/static/specs/os.yml +++ b/deployability/modules/allocation/static/specs/os.yml @@ -126,9 +126,15 @@ vagrant: macos-monterey-12.0.1-amd64: box: development/macos-monterey box_version: 0.0.0 + macos-ventura-13.4.1-amd64: + box: development/macos-ventura + box_version: 0.0.0 macos-ventura-13.4.1-arm64: box: macos-13 box_version: 0.0.0 + macos-sonoma-14.4.1-amd64: + box: development/macos-sonoma + box_version: 0.0.0 macos-sonoma-14.0-arm64: box: macos-14 box_version: 0.0.0 diff --git a/deployability/modules/allocation/static/templates/vagrant_black_mini.j2 b/deployability/modules/allocation/static/templates/vagrant_Virtual_box.j2 similarity index 71% rename from deployability/modules/allocation/static/templates/vagrant_black_mini.j2 rename to deployability/modules/allocation/static/templates/vagrant_Virtual_box.j2 index 870e3ea9ea..4643206504 100644 --- a/deployability/modules/allocation/static/templates/vagrant_black_mini.j2 +++ b/deployability/modules/allocation/static/templates/vagrant_Virtual_box.j2 @@ -3,10 +3,6 @@ Vagrant.configure("2") do |config| config.vm.network :private_network, type: "dhcp" config.vm.network "forwarded_port", guest: 22, host: "{{ config.port }}" - config.vm.provision "file", source: "~/.ssh/vagrant_rsa.pub", destination: "~/.ssh/vagrant_rsa.pub" - config.vm.provision "file", source: "~/.ssh/authorized_keys", destination: "~/.ssh/authorized_keys" - - config.vm.synced_folder ".", "/vagrant", disabled: true config.vm.provider "virtualbox" do |vb| diff --git a/deployability/modules/allocation/static/templates/vagrant_macStadium.j2 b/deployability/modules/allocation/static/templates/vagrant_parallels_arm.j2 old mode 100755 new mode 100644 similarity index 100% rename from deployability/modules/allocation/static/templates/vagrant_macStadium.j2 rename to deployability/modules/allocation/static/templates/vagrant_parallels_arm.j2 diff --git a/deployability/modules/allocation/static/templates/vagrant_parallels_intel.j2 b/deployability/modules/allocation/static/templates/vagrant_parallels_intel.j2 new file mode 100755 index 0000000000..6d539bef36 --- /dev/null +++ b/deployability/modules/allocation/static/templates/vagrant_parallels_intel.j2 @@ -0,0 +1,19 @@ +Vagrant.configure("2") do |config| + # Box image settings + config.vm.box = "{{ config.box }}" + + # VirtualBox specific settings + config.vm.provider "parallels" do |v| + v.memory = "4096" + v.cpus = "2" + v.name = "{{ config.name }}" + v.linked_clone = false + end + + # Network settings + #config.vm.network :private_network, type: "dhcp" + config.ssh.forward_agent = true + config.vm.network "forwarded_port", guest: 22, host: "{{ config.port }}" + #config.vm.network "public_network", bridge: "en0" + config.vm.synced_folder ".", "/vagrant", disabled: true +end diff --git a/deployability/modules/allocation/vagrant/helpers/vagrant_script.sh b/deployability/modules/allocation/vagrant/helpers/vagrant_script.sh new file mode 100644 index 0000000000..f91a83d611 --- /dev/null +++ b/deployability/modules/allocation/vagrant/helpers/vagrant_script.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Set the path to Vagrant directory +VAGRANT_DIR="/usr/local/bin" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Add Vagrant directory to PATH +export PATH=$PATH:$VAGRANT_DIR + +# Check if an argument is provided +if [ $# -eq 0 ] +then + echo "Usage: $0 [up | destroy | status | ...]" + exit 1 +fi + +if [ $1 == "destroy" ]; then + VAGRANT_CWD=$SCRIPT_DIR vagrant $1 -f +else + VAGRANT_CWD=$SCRIPT_DIR vagrant $1 +fi + +exit 0 \ No newline at end of file diff --git a/deployability/modules/allocation/vagrant/instance.py b/deployability/modules/allocation/vagrant/instance.py index f5bb5bfedc..4351b74f08 100755 --- a/deployability/modules/allocation/vagrant/instance.py +++ b/deployability/modules/allocation/vagrant/instance.py @@ -105,13 +105,10 @@ def delete(self) -> None: if str(self.host_identifier) == "macstadium": logger.debug(f"Deleting remote directory {self.host_instance_dir}") VagrantUtils.remote_command(f"sudo rm -rf {self.host_instance_dir}", self.remote_host_parameters) - logger.debug(f"Killing remote process on port {self.ssh_port}") - proccess = VagrantUtils.remote_command(f"sudo lsof -Pi :{self.ssh_port} -sTCP:LISTEN -t", self.remote_host_parameters) - VagrantUtils.remote_command(f"sudo kill -9 {proccess}", self.remote_host_parameters) - if str(self.host_identifier) == "black_mini": - logger.debug(f"Deleting remote directory {self.host_instance_dir}") - VagrantUtils.remote_command(f"sudo rm -rf {self.host_instance_dir}", self.remote_host_parameters) - + if self.arch == 'arm64': + logger.debug(f"Killing remote process on port {self.ssh_port}") + proccess = VagrantUtils.remote_command(f"sudo lsof -Pi :{self.ssh_port} -sTCP:LISTEN -t", self.remote_host_parameters) + VagrantUtils.remote_command(f"sudo kill -9 {proccess}", self.remote_host_parameters) def status(self) -> str: """ @@ -209,7 +206,7 @@ def __run_vagrant_command(self, command: str | list) -> str: if isinstance(command, str): command = [command] if self.platform == 'macos': - cmd = f"sudo VAGRANT_CWD={self.host_instance_dir} /usr/local/bin/vagrant " + ' '.join(command) + cmd = f"sudo {self.host_instance_dir}/vagrant_script.sh " + ' '.join(command) output = VagrantUtils.remote_command(cmd, self.remote_host_parameters) return output else: diff --git a/deployability/modules/allocation/vagrant/provider.py b/deployability/modules/allocation/vagrant/provider.py index 007eee5d50..89a1f41c98 100644 --- a/deployability/modules/allocation/vagrant/provider.py +++ b/deployability/modules/allocation/vagrant/provider.py @@ -92,6 +92,9 @@ def _create_instance(cls, base_dir: Path, params: CreationPayload, config: Vagra if platform == 'macos': vagrant_file = str(instance_dir) + '/Vagrantfile' VagrantUtils.remote_copy(vagrant_file, host_instance_dir, remote_host_parameters) + VagrantUtils.remote_copy(Path(__file__).parent.parent / 'vagrant' / 'helpers' / 'vagrant_script.sh', host_instance_dir, remote_host_parameters) + cmd = f"chmod 700 {host_instance_dir}/vagrant_script.sh" + VagrantUtils.remote_command(cmd, remote_host_parameters) instance_params = {} instance_params['instance_dir'] = instance_dir @@ -185,10 +188,13 @@ def __render_vagrantfile(cls, config: VagrantConfig) -> str: """ environment = Environment(loader=FileSystemLoader(cls.TEMPLATES_DIR)) if config.platform == 'macos': - if config.arch == 'arm64': - template = environment.get_template("vagrant_macStadium.j2") + if config.arch == 'amd64': + if config.box != 'development/macos-high-sierra' and config.box != 'development/macos-mojave' and config.box != 'development/macos-sierra' and config.box != 'development/macos-sierra_cmake': + template = environment.get_template("vagrant_parallels_intel.j2") + else: + template = environment.get_template("vagrant_Virtual_box.j2") else: - template = environment.get_template("vagrant_black_mini.j2") + template = environment.get_template("vagrant_parallels_arm.j2") else: template = environment.get_template("vagrant.j2") return template.render(config=config) From 47a2c61acb919f59a589f56c95129c8abc1c71df Mon Sep 17 00:00:00 2001 From: c-bordon Date: Thu, 2 May 2024 15:45:24 -0300 Subject: [PATCH 3/6] Removed commented configurations in intel vagrantfile --- .../allocation/static/templates/vagrant_parallels_intel.j2 | 2 -- .../modules/allocation/vagrant/helpers/vagrant_script.sh | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/deployability/modules/allocation/static/templates/vagrant_parallels_intel.j2 b/deployability/modules/allocation/static/templates/vagrant_parallels_intel.j2 index 6d539bef36..1af532acd1 100755 --- a/deployability/modules/allocation/static/templates/vagrant_parallels_intel.j2 +++ b/deployability/modules/allocation/static/templates/vagrant_parallels_intel.j2 @@ -11,9 +11,7 @@ Vagrant.configure("2") do |config| end # Network settings - #config.vm.network :private_network, type: "dhcp" config.ssh.forward_agent = true config.vm.network "forwarded_port", guest: 22, host: "{{ config.port }}" - #config.vm.network "public_network", bridge: "en0" config.vm.synced_folder ".", "/vagrant", disabled: true end diff --git a/deployability/modules/allocation/vagrant/helpers/vagrant_script.sh b/deployability/modules/allocation/vagrant/helpers/vagrant_script.sh index f91a83d611..236c77606f 100644 --- a/deployability/modules/allocation/vagrant/helpers/vagrant_script.sh +++ b/deployability/modules/allocation/vagrant/helpers/vagrant_script.sh @@ -10,8 +10,8 @@ export PATH=$PATH:$VAGRANT_DIR # Check if an argument is provided if [ $# -eq 0 ] then - echo "Usage: $0 [up | destroy | status | ...]" - exit 1 + echo "Usage: $0 [up | destroy | status | ...]" + exit 1 fi if [ $1 == "destroy" ]; then @@ -20,4 +20,4 @@ else VAGRANT_CWD=$SCRIPT_DIR vagrant $1 fi -exit 0 \ No newline at end of file +exit 0 From d3a89011a913f58b3291d93802944fcd1fcb677e Mon Sep 17 00:00:00 2001 From: c-bordon Date: Fri, 3 May 2024 09:12:24 -0300 Subject: [PATCH 4/6] Added new virtualbox validations --- deployability/modules/allocation/vagrant/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployability/modules/allocation/vagrant/provider.py b/deployability/modules/allocation/vagrant/provider.py index 89a1f41c98..dbfe05de53 100644 --- a/deployability/modules/allocation/vagrant/provider.py +++ b/deployability/modules/allocation/vagrant/provider.py @@ -189,7 +189,7 @@ def __render_vagrantfile(cls, config: VagrantConfig) -> str: environment = Environment(loader=FileSystemLoader(cls.TEMPLATES_DIR)) if config.platform == 'macos': if config.arch == 'amd64': - if config.box != 'development/macos-high-sierra' and config.box != 'development/macos-mojave' and config.box != 'development/macos-sierra' and config.box != 'development/macos-sierra_cmake': + if config.box != 'development/macos-high-sierra' and config.box != 'development/macos-mojave' and config.box != 'development/macos-sierra' and config.box != 'development/macos-sierra_cmake' and config.box != 'development/macos-sierra_gcc9': template = environment.get_template("vagrant_parallels_intel.j2") else: template = environment.get_template("vagrant_Virtual_box.j2") From f4fac3143aaf12a57439dcc0fde9e228f270fa1b Mon Sep 17 00:00:00 2001 From: c-bordon Date: Tue, 7 May 2024 14:31:31 -0300 Subject: [PATCH 5/6] macOS Intel available boxes updated --- deployability/modules/allocation/static/specs/os.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/deployability/modules/allocation/static/specs/os.yml b/deployability/modules/allocation/static/specs/os.yml index d62ad648cf..35980524d5 100644 --- a/deployability/modules/allocation/static/specs/os.yml +++ b/deployability/modules/allocation/static/specs/os.yml @@ -99,12 +99,6 @@ vagrant: box: generic/rocky9 box_version: 4.3.12 # Macos - macos-bigsur-11.0-amd64: - box: development/macos-big-sur - box_version: 0.0.0 - macos-catalina-10.15.1-amd64: - box: development/macos-catalina - box_version: 0.0.0 macos-highsierra-10.13.6-amd64: box: development/macos-high-sierra box_version: 0.0.0 @@ -123,9 +117,6 @@ vagrant: macos-monterey-12.6-arm64: box: macos-12 box_version: 0.0.0 - macos-monterey-12.0.1-amd64: - box: development/macos-monterey - box_version: 0.0.0 macos-ventura-13.4.1-amd64: box: development/macos-ventura box_version: 0.0.0 From b43a25b40037ea2e59ed03a32c5e832bc7fbe06b Mon Sep 17 00:00:00 2001 From: c-bordon Date: Tue, 7 May 2024 14:46:30 -0300 Subject: [PATCH 6/6] Improvement in virtualbox box validation --- deployability/modules/allocation/vagrant/provider.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployability/modules/allocation/vagrant/provider.py b/deployability/modules/allocation/vagrant/provider.py index 1b41962448..01105d35de 100644 --- a/deployability/modules/allocation/vagrant/provider.py +++ b/deployability/modules/allocation/vagrant/provider.py @@ -190,7 +190,8 @@ def __render_vagrantfile(cls, config: VagrantConfig) -> str: environment = Environment(loader=FileSystemLoader(cls.TEMPLATES_DIR)) if config.platform == 'macos': if config.arch == 'amd64': - if config.box != 'development/macos-high-sierra' and config.box != 'development/macos-mojave' and config.box != 'development/macos-sierra' and config.box != 'development/macos-sierra_cmake' and config.box != 'development/macos-sierra_gcc9': + virtualbox_boxes = ['development/macos-high-sierra', 'development/macos-mojave', 'development/macos-sierra', 'development/macos-sierra_cmake', 'development/macos-sierra_gcc9'] + if config.box not in virtualbox_boxes: template = environment.get_template("vagrant_parallels_intel.j2") else: template = environment.get_template("vagrant_Virtual_box.j2")