From bb0fa3458156c7b49194aa20498ce90f22646747 Mon Sep 17 00:00:00 2001 From: Ian Roquebert <72234714+gzpcho@users.noreply.github.com> Date: Tue, 13 Feb 2024 10:32:43 -0600 Subject: [PATCH 1/9] [deploy-agent] Return type hints in common folder (#1404) Add type hints --- deploy-agent/deployd/common/caller.py | 7 +- deploy-agent/deployd/common/config.py | 97 ++++++++++--------- deploy-agent/deployd/common/decorators.py | 5 +- deploy-agent/deployd/common/env_status.py | 8 +- deploy-agent/deployd/common/executor.py | 19 ++-- deploy-agent/deployd/common/helper.py | 11 ++- .../deployd/common/single_instance.py | 4 +- deploy-agent/deployd/common/stats.py | 69 ++++++------- deploy-agent/deployd/common/types.py | 18 ++-- deploy-agent/deployd/common/utils.py | 37 +++---- 10 files changed, 141 insertions(+), 134 deletions(-) diff --git a/deploy-agent/deployd/common/caller.py b/deploy-agent/deployd/common/caller.py index c0e33a1fd3..cbc008022b 100644 --- a/deploy-agent/deployd/common/caller.py +++ b/deploy-agent/deployd/common/caller.py @@ -15,7 +15,8 @@ import subprocess import traceback import logging -import time +import time +from typing import Optional, Tuple from future.utils import PY3 @@ -23,11 +24,11 @@ class Caller(object): - def __init__(self): + def __init__(self) -> None: pass @staticmethod - def call_and_log(cmd, **kwargs): + def call_and_log(cmd, **kwargs) -> Tuple[Optional[str], str, Optional[int]]: output = "" start = time.time() try: diff --git a/deploy-agent/deployd/common/config.py b/deploy-agent/deployd/common/config.py index e1f0199f74..369acca1ba 100644 --- a/deploy-agent/deployd/common/config.py +++ b/deploy-agent/deployd/common/config.py @@ -15,6 +15,7 @@ import logging import os +from typing import Any, List, Optional from deployd import __version__ @@ -34,7 +35,7 @@ class Config(object): _DEFAULT_CONFIG_SECTION = 'default_config' _configs = {} - def __init__(self, filenames=None, config_reader=None): + def __init__(self, filenames=None, config_reader=None) -> None: self._configs = {} self._filenames = None self._environ = {} @@ -56,10 +57,10 @@ def __init__(self, filenames=None, config_reader=None): print('Cannot read config files: {}'.format(self._filenames)) exit_abruptly(1) - def get_config_filename(self): + def get_config_filename(self) -> Optional[List[str]]: return self._filenames - def _get_deploy_type_from_opcode(self, opCode): + def _get_deploy_type_from_opcode(self, opCode) -> str: # TODO: Should use common.types.OpCode for next version if opCode == 'RESTART': return DeployType.RESTART @@ -70,7 +71,7 @@ def _get_deploy_type_from_opcode(self, opCode): else: return DeployType.REGULAR - def update_variables(self, deploy_status): + def update_variables(self, deploy_status) -> None: if not deploy_status: return @@ -134,7 +135,7 @@ def update_variables(self, deploy_status): self._environ['BUILDS_DIR'] = self.get_builds_directory() os.environ.update(self._environ) - def get_var(self, var_name, default_value=None): + def get_var(self, var_name, default_value=None) -> Any: try: if self._configs and var_name in self._configs: return self._configs[var_name] @@ -145,17 +146,17 @@ def get_var(self, var_name, default_value=None): return default_value raise DeployConfigException('{} cannot be found.'.format(var_name)) - def get_intvar(self, var_name, default_value=None): + def get_intvar(self, var_name, default_value=None) -> int: return int(self.get_var(var_name, default_value)) - def get_target(self): + def get_target(self) -> Optional[str]: target_default_dir = self.get_var("target_default_dir", "/tmp") if not (self._configs and self._configs.get('target')): return os.path.join(target_default_dir, self._environ['ENV_NAME']) return self._configs.get('target') - def get_subprocess_log_name(self): + def get_subprocess_log_name(self) -> str: if 'ENV_NAME' in self._environ: return '{}/{}.log'.format(self.get_log_directory(), self._environ['ENV_NAME']) else: @@ -169,88 +170,88 @@ def get_script_directory(self): else: return script_dir - def get_agent_directory(self): + def get_agent_directory(self) -> str: return self.get_var("deploy_agent_dir", "/tmp/deployd/") - def get_env_status_fn(self): + def get_env_status_fn(self) -> str: return os.path.join(self.get_agent_directory(), "env_status") - def get_host_info_fn(self): + def get_host_info_fn(self) -> str: return os.path.join(self.get_agent_directory(), "host_info") - def get_builds_directory(self): + def get_builds_directory(self) -> str: return self.get_var("builds_dir", "/tmp/deployd/builds") - def get_log_directory(self): + def get_log_directory(self) -> str: return self.get_var("log_directory", "/tmp/deployd/logs") - def get_user_role(self): + def get_user_role(self) -> str: import getpass return self.get_var("user_role", getpass.getuser()) - def get_restful_service_url(self): + def get_restful_service_url(self) -> str: return self.get_var('teletraan_service_url', 'http://localhost:8080') - def get_restful_service_version(self): + def get_restful_service_version(self) -> str: return self.get_var('teletraan_service_version', 'v1') - def get_restful_service_token(self): + def get_restful_service_token(self) -> str: return self.get_var('teletraan_service_token', '') # aws specific configuration - def get_aws_access_key(self): + def get_aws_access_key(self) -> Optional[str]: return self.get_var('aws_access_key_id', None) - def get_aws_access_secret(self): + def get_aws_access_secret(self) -> Optional[str]: return self.get_var('aws_secret_access_key', None) # agent process configs - def get_agent_ping_interval(self): + def get_agent_ping_interval(self) -> int: return self.get_intvar('min_running_time', 60) - def get_subprocess_running_timeout(self): + def get_subprocess_running_timeout(self) -> int: return self.get_intvar('process_timeout', 1800) - def get_subprocess_terminate_timeout(self): + def get_subprocess_terminate_timeout(self) -> int: return self.get_intvar('termination_timeout', 30) - def get_subproces_max_retry(self): + def get_subproces_max_retry(self) -> int: return self.get_intvar('max_retry', 3) - def get_subprocess_max_log_bytes(self): + def get_subprocess_max_log_bytes(self) -> int: return self.get_intvar('max_tail_bytes', 10240) - def get_subprocess_max_sleep_interval(self): + def get_subprocess_max_sleep_interval(self) -> int: return self.get_intvar('max_sleep_interval', 60) - def get_subprocess_poll_interval(self): + def get_subprocess_poll_interval(self) -> int: return self.get_intvar("process_wait_interval", 2) - def get_backoff_factor(self): + def get_backoff_factor(self) -> int: return self.get_intvar("back_off_factor", 2) - def get_num_builds_retain(self): + def get_num_builds_retain(self) -> int: return self.get_intvar("num_builds_to_retain", 2) - def respect_puppet(self): + def respect_puppet(self) -> int: return self.get_intvar("respect_puppet", 0) - def get_puppet_state_file_path(self): + def get_puppet_state_file_path(self) -> Optional[str]: return self.get_var("puppet_file_path", None) - def get_puppet_summary_file_path(self): + def get_puppet_summary_file_path(self) -> str: return self.get_var("puppet_summary_file_path", "/var/cache/puppet/state/last_run_summary.yaml") - def get_puppet_exit_code_file_path(self): + def get_puppet_exit_code_file_path(self) -> str: return self.get_var("puppet_exit_code_file_path", "/var/log/puppet/puppet_exit_code") - def get_daemon_sleep_time(self): + def get_daemon_sleep_time(self) -> int: return self.get_intvar("daemon_sleep_time", 30) - def get_init_sleep_time(self): + def get_init_sleep_time(self) -> int: return self.get_intvar("init_sleep_time", 50) - def get_log_level(self): + def get_log_level(self) -> int: log_level = self.get_var("log_level", 'DEBUG') if log_level == "INFO": return logging.INFO @@ -258,41 +259,41 @@ def get_log_level(self): return logging.ERROR return logging.DEBUG - def get_facter_id_key(self): + def get_facter_id_key(self) -> Optional[str]: return self.get_var('agent_id_key', None) - def get_facter_ip_key(self): + def get_facter_ip_key(self) -> Optional[str]: return self.get_var('agent_ip_key', None) - def get_facter_name_key(self): + def get_facter_name_key(self) -> Optional[str]: return self.get_var('agent_name_key', None) - def get_facter_group_key(self): + def get_facter_group_key(self) -> Optional[str]: return self.get_var('agent_group_key', None) - def get_verify_https_certificate(self): + def get_verify_https_certificate(self) -> Optional[str]: return self.get_var('verify_https_certificate', 'False') - def get_deploy_agent_version(self): + def get_deploy_agent_version(self) -> str: return self.get_var('deploy_agent_version', __version__) - def get_facter_az_key(self): + def get_facter_az_key(self) -> Optional[str]: return self.get_var('availability_zone_key', None) - def get_facter_ec2_tags_key(self): + def get_facter_ec2_tags_key(self) -> Optional[str]: return self.get_var('ec2_tags_key', None) - def get_facter_asg_tag_key(self): + def get_facter_asg_tag_key(self) -> Optional[str]: return self.get_var('autoscaling_tag_key', None) - def get_stage_type_key(self): + def get_stage_type_key(self) -> Optional[str]: return self.get_var('stage_type_key', None) - def get_facter_account_id_key(self): + def get_facter_account_id_key(self) -> str: return self.get_var('account_id_key', 'ec2_metadata.identity-credentials.ec2.info') - def get_http_download_allow_list(self): + def get_http_download_allow_list(self) -> List: return self.get_var('http_download_allow_list', []) - def get_s3_download_allow_list(self): + def get_s3_download_allow_list(self) -> List: return self.get_var('s3_download_allow_list', []) diff --git a/deploy-agent/deployd/common/decorators.py b/deploy-agent/deployd/common/decorators.py index 123a25a3dc..5965039966 100644 --- a/deploy-agent/deployd/common/decorators.py +++ b/deploy-agent/deployd/common/decorators.py @@ -13,9 +13,10 @@ # limitations under the License. import time +from typing import Callable -def singleton(non_singleton_cls): +def singleton(non_singleton_cls) -> Callable: """Decorator to make sure there is only one instance of a class. Args: cls: A class. @@ -36,7 +37,7 @@ def getinstance(*args, **kwargs): return getinstance -def retry(ExceptionToCheck, tries=4, delay=3, backoff=2): +def retry(ExceptionToCheck, tries=4, delay=3, backoff=2) -> Callable: """Retry calling the decorated function using an exponential backoff. Similar to utils.decorators.retry() but customized slightly for operations tools. diff --git a/deploy-agent/deployd/common/env_status.py b/deploy-agent/deployd/common/env_status.py index 2f9c0c30c6..02d36e97db 100644 --- a/deploy-agent/deployd/common/env_status.py +++ b/deploy-agent/deployd/common/env_status.py @@ -27,12 +27,12 @@ class EnvStatus(object): - def __init__(self, status_fn): + def __init__(self, status_fn) -> None: self._status_fn = status_fn self._lock_fn = '{}.lock'.format(self._status_fn) self._lock = lockfile.FileLock(self._lock_fn) - def load_envs(self): + def load_envs(self) -> dict: """ open up config file validate that the service selected exists @@ -52,7 +52,7 @@ def load_envs(self): finally: return envs - def _touch_or_rm_host_type_file(self, envs, host_type, directory='/var/run/'): + def _touch_or_rm_host_type_file(self, envs, host_type, directory='/var/run/') -> None: """Touches or removes the identity file for the host type. For now, a host type could be 'canary'. """ @@ -74,7 +74,7 @@ def _touch_or_rm_host_type_file(self, envs, host_type, directory='/var/run/'): os.remove(file_path) log.debug('Removed {}.'.format(file_path)) - def dump_envs(self, envs): + def dump_envs(self, envs) -> bool: try: json_data = {} if envs: diff --git a/deploy-agent/deployd/common/executor.py b/deploy-agent/deployd/common/executor.py index 7d651e0c64..90b0736b8a 100644 --- a/deploy-agent/deployd/common/executor.py +++ b/deploy-agent/deployd/common/executor.py @@ -20,6 +20,7 @@ import stat import time import traceback +from typing import Tuple from future.utils import PY3 @@ -29,7 +30,7 @@ class Executor(object): - def __init__(self, callback=None, config=None): + def __init__(self, callback=None, config=None) -> None: self._ping_server = callback if not config: return @@ -37,7 +38,7 @@ def __init__(self, callback=None, config=None): self._config = config self.update_configs(config) - def update_configs(self, config): + def update_configs(self, config) -> None: self.LOG_FILENAME = config.get_subprocess_log_name() self.MAX_RUNNING_TIME = config.get_subprocess_running_timeout() self.MIN_RUNNING_TIME = config.get_agent_ping_interval() @@ -53,14 +54,14 @@ def update_configs(self, config): self.MAX_RUNNING_TIME, self.MAX_RETRY)) - def get_subprocess_output(self, fd, file_pos): + def get_subprocess_output(self, fd, file_pos) -> str: curr_pos = fd.tell() fd.seek(file_pos, 0) output = fd.read() fd.seek(curr_pos, 0) return output[-(self.MAX_TAIL_BYTES+1):-1] - def run_cmd(self, cmd, **kw): + def run_cmd(self, cmd, **kw) -> DeployReport: if not isinstance(cmd, list): cmd = cmd.split(' ') cmd_str = ' '.join(cmd) @@ -168,7 +169,7 @@ def run_cmd(self, cmd, **kw): deploy_report.status_code = AgentStatus.TOO_MANY_RETRY return deploy_report - def ping_server_if_possible(self, start, cmd_str, deploy_report): + def ping_server_if_possible(self, start, cmd_str, deploy_report) -> Tuple[datetime.datetime, DeployReport]: now = datetime.datetime.now() processed_time = (now - start).seconds log.debug("start: {}, now: {}, process: {}".format(start, now, processed_time)) @@ -184,20 +185,20 @@ def ping_server_if_possible(self, start, cmd_str, deploy_report): return start, deploy_report - def _get_sleep_interval(self, start, interval): + def _get_sleep_interval(self, start, interval) -> int: now = datetime.datetime.now() max_sleep_seconds = self.MIN_RUNNING_TIME - (now - start).seconds return min(interval, max(max_sleep_seconds, 1)) @staticmethod - def _kill_process(process): + def _kill_process(process) -> None: try: log.info('Kill currently running process') os.killpg(process.pid, signal.SIGKILL) except Exception as e: log.debug('Failed to kill process: {}'.format(e)) - def _graceful_shutdown(self, process): + def _graceful_shutdown(self, process) -> None: try: log.info('Gracefully shutdown currently running process with timeout {}'.format(self.TERMINATE_TIMEOUT)) os.killpg(process.pid, signal.SIGTERM) @@ -213,7 +214,7 @@ def _graceful_shutdown(self, process): log.debug('Failed to gracefully shutdown: {}'.format(e)) Executor._kill_process(process) - def execute_command(self, script): + def execute_command(self, script) -> DeployReport: try: deploy_step = os.getenv('DEPLOY_STEP') if not os.path.exists(self._config.get_script_directory()): diff --git a/deploy-agent/deployd/common/helper.py b/deploy-agent/deployd/common/helper.py index 60c63b3175..046a897da5 100644 --- a/deploy-agent/deployd/common/helper.py +++ b/deploy-agent/deployd/common/helper.py @@ -16,16 +16,17 @@ import logging import os import shutil +from typing import Generator, List, Optional, Tuple log = logging.getLogger(__name__) class Helper(object): - def __init__(self, config=None): + def __init__(self, config=None) -> None: self._config = config - def builds_available_locally(self, builds_dir, env_name): + def builds_available_locally(self, builds_dir, env_name) -> List: """Returns a list of (build, timestamp) that we have installed.""" builds = [] try: @@ -46,7 +47,7 @@ def builds_available_locally(self, builds_dir, env_name): return builds @staticmethod - def get_build_id(filename, env_name): + def get_build_id(filename, env_name) -> Tuple[bool, Optional[str]]: """ Extract build id from the file name In downloader.py, we have the following name convenion @@ -59,7 +60,7 @@ def get_build_id(filename, env_name): return False, None @staticmethod - def get_stale_builds(build_timestamps, num_builds_to_retain=2): + def get_stale_builds(build_timestamps, num_builds_to_retain=2) -> Generator: """ Given a list of (build, timestamp) tuples, determine which are stale. @@ -80,7 +81,7 @@ def get_stale_builds(build_timestamps, num_builds_to_retain=2): yielded_builds += 1 @staticmethod - def clean_package(base_dir, build, build_name): + def clean_package(base_dir, build, build_name) -> None: """ Clean a package: :param base_dir: builds dir diff --git a/deploy-agent/deployd/common/single_instance.py b/deploy-agent/deployd/common/single_instance.py index b54d57b434..33713f8939 100644 --- a/deploy-agent/deployd/common/single_instance.py +++ b/deploy-agent/deployd/common/single_instance.py @@ -27,7 +27,7 @@ class SingleInstance(object): - def __init__(self): + def __init__(self) -> None: # Establish lock file settings appname = 'deploy-agent' lockfile_name = '.{}.lock'.format(appname) @@ -60,7 +60,7 @@ def __init__(self): os.close(lockfile_fd) utils.exit_abruptly(1) - def _create_lock_dir(self): + def _create_lock_dir(self) -> None: if PY3: os.makedirs(LOCKFILE_DIR, exist_ok=True) else: diff --git a/deploy-agent/deployd/common/stats.py b/deploy-agent/deployd/common/stats.py index bdaa4d0378..3dd02c62d0 100644 --- a/deploy-agent/deployd/common/stats.py +++ b/deploy-agent/deployd/common/stats.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +from typing import Generator, Optional, Union from deployd import __version__, IS_PINTEREST, METRIC_PORT_HEALTH, METRIC_CACHE_PATH, STATSBOARD_URL import timeit import socket @@ -53,7 +54,7 @@ def __exit__(self, stats, sample_rate=1.0, tags=None): class TimingOnlyStatClient: """ timing only stat client in order to use caching """ @staticmethod - def timing(*args, **kwargs): + def timing(*args, **kwargs) -> None: client = MetricClient() return client.send_context_timer(*args, **kwargs) @@ -70,7 +71,7 @@ def create_stats_timer(name, sample_rate=1.0, tags=None): return DefaultStatsdTimer() -def create_sc_timing(name, value, sample_rate=1.0, tags=None): +def create_sc_timing(name, value, sample_rate=1.0, tags=None) -> None: if IS_PINTEREST: mtype = 'timing' client = MetricClient() @@ -83,7 +84,7 @@ def create_sc_timing(name, value, sample_rate=1.0, tags=None): return -def create_sc_increment(name, sample_rate=1.0, tags=None): +def create_sc_increment(name, sample_rate=1.0, tags=None) -> None: if IS_PINTEREST: mtype = 'increment' client = MetricClient() @@ -95,7 +96,7 @@ def create_sc_increment(name, sample_rate=1.0, tags=None): return -def create_sc_gauge(name, value, sample_rate=1.0, tags=None): +def create_sc_gauge(name, value, sample_rate=1.0, tags=None) -> None: if IS_PINTEREST: mtype = 'gauge' client = MetricClient() @@ -107,7 +108,7 @@ def create_sc_gauge(name, value, sample_rate=1.0, tags=None): else: return -def send_statsboard_metric(name, value, tags=None): +def send_statsboard_metric(name, value, tags=None) -> None: tags['host'] = socket.gethostname() tags_params = [f"{tag}={tags[tag]}" for tag in tags] tags_str = ",".join(tags_params) @@ -123,7 +124,7 @@ def send_statsboard_metric(name, value, tags=None): class MetricCacheConfigurationError(ValueError): """ Raised when MetricCache has missing configuration """ - def __init__(self, name, value): + def __init__(self, name, value) -> None: msg = '{} is {}'.format(name, value) super(MetricCacheConfigurationError, self).__init__(msg) @@ -132,7 +133,7 @@ class MetricCache: """ local cache for metrics creates empty cache file """ - def __init__(self, path=METRIC_CACHE_PATH): + def __init__(self, path=METRIC_CACHE_PATH) -> None: if not path: raise MetricCacheConfigurationError('path', path) self.path = path @@ -142,13 +143,13 @@ def __init__(self, path=METRIC_CACHE_PATH): if not self.exists(): self.truncate() - def limit(self): + def limit(self) -> bool: """ check to see if cache file has exceeded maximum size return: bool """ return os.path.getsize(self.path) > self.max_size - def exists(self): + def exists(self) -> bool: """ cache file exists and is read/write return: bool """ @@ -158,13 +159,13 @@ def exists(self): return True return False - def is_empty(self): + def is_empty(self) -> bool: """ check if the cache not empty return: bool """ return not os.stat(self.path).st_size > 0 - def read(self): + def read(self) -> Generator: """ read metrics from cache, then delete return: generator """ @@ -172,7 +173,7 @@ def read(self): for line in fh: yield Stat(ins=line) - def write(self, output): + def write(self, output) -> None: """ write metrics to cache file respecting max cache size appends newline to metric """ @@ -183,7 +184,7 @@ def write(self, output): with open(self.path, 'a') as fh: fh.write('{}\n'.format(output)) - def truncate(self): + def truncate(self) -> None: """ purge cache file """ with open(self.path, 'w') as fh: fh.truncate() @@ -194,7 +195,7 @@ class Stat: supports all methods for stats """ - def __init__(self, mtype=None, name=None, value=None, sample_rate=None, tags=None, ins=None): + def __init__(self, mtype=None, name=None, value=None, sample_rate=None, tags=None, ins=None) -> None: self.mtype = mtype self.name = name self.value = value @@ -207,7 +208,7 @@ def __init__(self, mtype=None, name=None, value=None, sample_rate=None, tags=Non # python2 support self.JSONDecodeError = ValueError - def serialize(self): + def serialize(self) -> str: """ serialize for cache writing """ obj = dict() obj['mtype'] = self.mtype @@ -219,7 +220,7 @@ def serialize(self): obj['tags'] = self.tags return json.dumps(obj) - def _deserialize(self): + def _deserialize(self) -> bool: """ read in json, setting defaults for a stat return: bool """ @@ -233,7 +234,7 @@ def _deserialize(self): return True return False - def deserialize(self, ins=None): + def deserialize(self, ins=None) -> bool: """ attempt to deserialize :param: ins json as str return: bool, False on error @@ -253,7 +254,7 @@ def deserialize(self, ins=None): class MetricClientConfigurationError(ValueError): """ Raised when MetricClient has missing configuration """ - def __init__(self, name, value): + def __init__(self, name, value) -> None: msg = '{} is {}'.format(name, value) super(MetricClientConfigurationError, self).__init__(msg) @@ -261,7 +262,7 @@ def __init__(self, name, value): class MetricClient: """ metrics client wrapper, enables disk cache """ - def __init__(self, port=METRIC_PORT_HEALTH, cache_path=METRIC_CACHE_PATH): + def __init__(self, port=METRIC_PORT_HEALTH, cache_path=METRIC_CACHE_PATH) -> None: if not port: raise MetricClientConfigurationError('port', port) self.port = port @@ -269,7 +270,7 @@ def __init__(self, port=METRIC_PORT_HEALTH, cache_path=METRIC_CACHE_PATH): self.stat = None @staticmethod - def _add_default_tags(tags=None): + def _add_default_tags(tags=None) -> Optional[dict]: """ add default tags to stats :param: tags as dict return: dict @@ -283,7 +284,7 @@ def _add_default_tags(tags=None): return tags @staticmethod - def _parse_stat(mtype=None, name=None, value=None, sample_rate=None, tags=None): + def _parse_stat(mtype=None, name=None, value=None, sample_rate=None, tags=None) -> Stat: """ return Stat for given kwargs """ return Stat(mtype=mtype, name=name, @@ -291,7 +292,7 @@ def _parse_stat(mtype=None, name=None, value=None, sample_rate=None, tags=None): sample_rate=sample_rate, tags=tags) - def _send_mtype(self): + def _send_mtype(self) -> None: """ send metric to sc using corrected calls per metric type """ @@ -316,14 +317,14 @@ def _send_mtype(self): msg = 'encountered unsupported mtype:{} while sending name:{}, value:{}, sample_rate:{}, tags:{}' log.error(msg.format(self.stat.mtype, self.stat.name, self.stat.value, self.stat.sample_rate, self.stat.tags)) - def _send(self): + def _send(self) -> None: """ send metric to sc """ try: self._send_mtype() except Exception as error: log.error('unable to send metric: {}'.format(error)) - def _flush_cache(self): + def _flush_cache(self) -> None: """ read from cache, send every metric, truncate cache """ log.warning('flushing metrics from cache') for stat in self.cache.read(): @@ -336,13 +337,13 @@ def _flush_cache(self): # truncate cache file after flush self.cache.truncate() - def send_context_timer(self, name, value, sample_rate=None, tags=None): + def send_context_timer(self, name, value, sample_rate=None, tags=None) -> None: """ convert a context_timer to timing for cacheability """ self.send(mtype='timing', name=name, value=value, sample_rate=sample_rate, tags=tags) - def send(self, mtype=None, name=None, value=None, sample_rate=None, tags=None): + def send(self, mtype=None, name=None, value=None, sample_rate=None, tags=None) -> None: """ add default tags, send metric, write to, or flush cache depending on health check """ tags = self._add_default_tags(tags) @@ -358,7 +359,7 @@ def send(self, mtype=None, name=None, value=None, sample_rate=None, tags=None): stat = self._parse_stat(mtype=mtype, name=name, value=value, sample_rate=sample_rate, tags=tags) self.cache.write(stat.serialize()) - def is_healthy(self): + def is_healthy(self) -> bool: """ health-check by connecting to local IPv4 TCP listening socket return: bool """ @@ -380,13 +381,13 @@ def is_healthy(self): class TimeElapsed: """ keep track of elapsed time in seconds """ - def __init__(self): + def __init__(self) -> None: self._time_start = self._timer() self._time_now = None self._time_elapsed = float(0) self._time_pause = None - def get(self): + def get(self) -> Union[float, int]: """ total elapsed running time, accuracy in seconds return: int """ @@ -397,7 +398,7 @@ def get(self): self._time_start = self._time_now return int(self._time_elapsed) - def _is_paused(self): + def _is_paused(self) -> bool: """ timer pause state return: bool """ @@ -406,13 +407,13 @@ def _is_paused(self): return False @staticmethod - def _timer(): + def _timer() -> float: """ timer in seconds return: float """ return timeit.default_timer() - def since_pause(self): + def since_pause(self) -> float: """ time elapsed since pause return: float """ @@ -420,14 +421,14 @@ def since_pause(self): return float(self._timer() - self._time_pause) return float(0) - def pause(self): + def pause(self) -> None: """ pause timer if not paused return: None """ if not self._is_paused(): self._time_pause = self._timer() - def resume(self): + def resume(self) -> None: """ resume timer if paused return: None """ diff --git a/deploy-agent/deployd/common/types.py b/deploy-agent/deployd/common/types.py index 0e8009bf04..3dd9e9037f 100644 --- a/deploy-agent/deployd/common/types.py +++ b/deploy-agent/deployd/common/types.py @@ -75,7 +75,7 @@ class OpCode(object): class DeployReport(object): def __init__(self, status_code=AgentStatus.UNKNOWN, - error_code=0, output_msg=None, retry_times=0): + error_code=0, output_msg=None, retry_times=0) -> None: self.status_code = status_code self.error_code = error_code self.output_msg = output_msg @@ -91,7 +91,7 @@ class PingStatus(object): class BuildInfo(object): def __init__(self, commit, build_url, build_id, build_name=None, - build_repo=None, build_branch=None): + build_repo=None, build_branch=None) -> None: self.build_commit = commit self.build_url = build_url self.build_id = build_id @@ -99,7 +99,7 @@ def __init__(self, commit, build_url, build_id, build_name=None, self.build_repo = build_repo self.build_branch = build_branch - def __eq__(self, other): + def __eq__(self, other) -> bool: if other: return self.__dict__ == other.__dict__ else: @@ -115,7 +115,7 @@ class DeployStatus(object): script_variables = None is_docker = None - def __init__(self, response=None, json_value=None): + def __init__(self, response=None, json_value=None) -> None: if response: self.update_by_response(response) elif json_value: @@ -128,7 +128,7 @@ def __eq__(self, other): self.op_code == other.op_code # update the deploy status by the current ping response from teletraan server - def update_by_response(self, response): + def update_by_response(self, response) -> None: deploy_goal = response.deployGoal self.op_code = response.opCode if not self.report: @@ -161,7 +161,7 @@ def update_by_response(self, response): self.runtime_config = dict(deploy_goal.config) # update the env status with current deploy report - def update_by_deploy_report(self, deploy_report): + def update_by_deploy_report(self, deploy_report) -> None: # aborted by server happens when if deploy_report.status_code == AgentStatus.ABORTED_BY_SERVER: self.report.status = AgentStatus.UNKNOWN @@ -174,7 +174,7 @@ def update_by_deploy_report(self, deploy_report): self.report.errorMessage = None self.report.failCount = deploy_report.retry_times - def load_from_json(self, json_value): + def load_from_json(self, json_value) -> None: report = json_value.get('report') build_info = json_value.get('build_info') if report: @@ -194,7 +194,7 @@ def load_from_json(self, json_value): else: self.op_code = op_code - def to_json(self): + def to_json(self) -> dict: json = {} for key, value in self.__dict__.items(): if type(value) is dict: @@ -206,7 +206,7 @@ def to_json(self): json[key] = value return json - def __str__(self): + def __str__(self) -> str: return str(self.to_json()) diff --git a/deploy-agent/deployd/common/utils.py b/deploy-agent/deployd/common/utils.py index 04787f4593..1bc5c662c2 100644 --- a/deploy-agent/deployd/common/utils.py +++ b/deploy-agent/deployd/common/utils.py @@ -22,6 +22,7 @@ import sys import traceback import subprocess +from typing import Any, Optional, Union import yaml @@ -34,7 +35,7 @@ # noinspection PyProtectedMember -def exit_abruptly(status=0): +def exit_abruptly(status=0) -> None: """Exit method that just quits abruptly. Helps with KeyError issues. @@ -48,7 +49,7 @@ def exit_abruptly(status=0): os._exit(status) -def touch(fname, times=None): +def touch(fname, times=None) -> None: try: with open(fname, 'a'): os.utime(fname, times) @@ -56,7 +57,7 @@ def touch(fname, times=None): log.error('Failed touching host type file {}'.format(fname)) -def hash_file(filepath): +def hash_file(filepath) -> str: """ hash the file content :param filepath: the full path of the file :return:the sha1 of the file data @@ -68,7 +69,7 @@ def hash_file(filepath): # steal from http://stackoverflow.com/questions/132058/ # showing-the-stack-trace-from-a-running-python-application # use : sudo kill -SIGUSR1 $pid to trigger the debug -def debug(sig, frame): +def debug(sig, frame) -> None: """Interrupt running process, and provide a python prompt for interactive debugging.""" d = {'_frame': frame} # Allow access to frame object. @@ -81,11 +82,11 @@ def debug(sig, frame): i.interact(message) -def listen(): +def listen() -> None: signal.signal(signal.SIGUSR1, debug) # Register handler -def mkdir_p(path): +def mkdir_p(path) -> None: try: os.makedirs(path) except OSError as ex: @@ -96,7 +97,7 @@ def mkdir_p(path): raise -def uptime(): +def uptime() -> int: """ return int: seconds of uptime in int, default 0 """ sec = 0 if sys.platform.startswith('linux'): @@ -106,19 +107,19 @@ def uptime(): return sec -def ensure_dirs(config): +def ensure_dirs(config) -> None: # make sure deployd directories exist mkdir_p(config.get_builds_directory()) mkdir_p(config.get_agent_directory()) mkdir_p(config.get_log_directory()) -def is_first_run(config): +def is_first_run(config) -> bool: env_status_file = config.get_env_status_fn() return not os.path.exists(env_status_file) -def check_prereqs(config): +def check_prereqs(config) -> bool: """ Check prerequisites before deploy agent can run @@ -140,7 +141,7 @@ def check_prereqs(config): return True -def get_puppet_exit_code(config): +def get_puppet_exit_code(config) -> Union[str, int]: """ Get puppet exit code from the corresponding file @@ -157,7 +158,7 @@ def get_puppet_exit_code(config): return exit_code -def load_puppet_summary(config): +def load_puppet_summary(config) -> dict: """ Load last_run_summary yaml file, parse results @@ -174,7 +175,7 @@ def load_puppet_summary(config): return summary -def check_first_puppet_run_success(config): +def check_first_puppet_run_success(config) -> bool: """ Check first puppet run success from exit code and last run summary @@ -199,7 +200,7 @@ def check_first_puppet_run_success(config): return puppet_failures == 0 -def get_info_from_facter(keys): +def get_info_from_facter(keys) -> Optional[dict]: try: time_facter = TimeElapsed() # increment stats - facter calls @@ -221,7 +222,7 @@ def get_info_from_facter(keys): return None -def redeploy_check(labels, service, redeploy): +def redeploy_check(labels, service, redeploy) -> int: max_retry = REDEPLOY_MAX_RETRY for label in labels: if "redeploy_max_retry" in label: @@ -230,7 +231,7 @@ def redeploy_check(labels, service, redeploy): return redeploy + 1 return 0 -def get_container_health_info(commit, service, redeploy): +def get_container_health_info(commit, service, redeploy) -> Optional[str]: try: log.info(f"Get health info for container with commit {commit}") result = [] @@ -272,7 +273,7 @@ def get_container_health_info(commit, service, redeploy): return None -def get_telefig_version(): +def get_telefig_version() -> Optional[str]: if not IS_PINTEREST: return None try: @@ -286,7 +287,7 @@ def get_telefig_version(): log.error("Error when fetching teletraan configure manager version") return None -def check_not_none(arg, msg=None): +def check_not_none(arg, msg=None) -> Any: if arg is None: raise ValueError(msg) return arg From 9c79b445b04b300f5a2bded923d6709a0ccfead8 Mon Sep 17 00:00:00 2001 From: Omar Date: Tue, 13 Feb 2024 09:39:39 -0800 Subject: [PATCH 2/9] Ensure `delivery_type` / `stage_type` are in sync (#1450) **Summary** Enforce that the `delivery_type` (value passed in by deployment pipeline) needs to be in sync with the `stage_type`. If they are not in sync, throw an error so that users can take action and resolve the inconsistency. **Caveats** In case the `delivery_type` / `stage_type` is out of sync, this could cause deployment failures. These failures should not happen frequently and are currently expected to be resolved manually. **Test Plan** Verified there have been no inconsistencies in the last week. --- .../deployservice/exception/TeletaanInternalException.java | 4 ++-- .../com/pinterest/deployservice/handler/DeployHandler.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/exception/TeletaanInternalException.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/exception/TeletaanInternalException.java index c76ad64dc4..ff6a4c5883 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/exception/TeletaanInternalException.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/exception/TeletaanInternalException.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/DeployHandler.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/DeployHandler.java index 11be3e9ad7..e7029b7415 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/DeployHandler.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/DeployHandler.java @@ -433,8 +433,11 @@ public String deploy(EnvironBean envBean, String buildId, String desc, String de // When the delivery_type is different with stage_type and stage_type is DEFAULT, we update the stage_type; if (StringUtils.isNotBlank(deliveryType) && !envBean.getStage_type().toString().equals(deliveryType)) { if (envBean.getStage_type() != EnvType.DEFAULT) { - LOG.error("The delivery type {} is different with the stage type {} for {}/{}", + String errorMessage = String.format( + "The delivery type %s is different with the stage type %s for %s/%s", deliveryType, envBean.getStage_type(), envBean.getEnv_name(), envBean.getStage_name()); + LOG.error(errorMessage); + throw new TeletaanInternalException(Response.Status.CONFLICT, errorMessage); } else { EnvType type = EnvType.valueOf(deliveryType); envBean.setStage_type(type); From 89498d495101d7f729d0d46f2c3ce3da06aa86b6 Mon Sep 17 00:00:00 2001 From: knguyen100000010 <63071572+knguyen100000010@users.noreply.github.com> Date: Tue, 13 Feb 2024 12:43:09 -0500 Subject: [PATCH 3/9] Prevent alarm created without scaling policy (#1448) * refuse to create alarm without scaling policy * add messages --- .../deploy_board/templates/groups/asg_config.html | 10 ++++++++-- deploy-board/deploy_board/webapp/group_view.py | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/deploy-board/deploy_board/templates/groups/asg_config.html b/deploy-board/deploy_board/templates/groups/asg_config.html index 603cc4bdb1..bd50802492 100644 --- a/deploy-board/deploy_board/templates/groups/asg_config.html +++ b/deploy-board/deploy_board/templates/groups/asg_config.html @@ -16,7 +16,6 @@ {% include "groups/group_side_panel.html" %} - {% endblock %} {% block new-builds-panel %} @@ -26,7 +25,14 @@ {% block main %} - +{% if storage %} + {% for message in storage %} + + {% endfor %} +{% endif %} {% if not is_cmp %}
diff --git a/deploy-board/deploy_board/webapp/group_view.py b/deploy-board/deploy_board/webapp/group_view.py index 3374de52eb..3e952c2f39 100644 --- a/deploy-board/deploy_board/webapp/group_view.py +++ b/deploy-board/deploy_board/webapp/group_view.py @@ -20,6 +20,7 @@ from django.template.loader import render_to_string from django.http import HttpResponse from django.contrib import messages +from django.contrib.messages import get_messages import json import logging import traceback @@ -794,6 +795,13 @@ def add_alarms(request, group_name): alarm_info["metricSource"] = params["awsMetrics"] alarm_info["groupName"] = group_name + if len(alarm_info["scalingPolicies"]) == 0: + # Technically, an alarm can be created without an action (e.g. scaling policy) + # However, in the current context, an alarm must have an associated scaling policy. + # In this case, there is no scaling policy, so refuse to create new alarm. + messages.add_message(request, messages.ERROR, 'Alarm could not be created because there was no {} policy.'.format(policy_type)) + return redirect("/groups/{}/config/".format(group_name)) + autoscaling_groups_helper.add_alarm(request, group_name, [alarm_info]) return redirect("/groups/{}/config/".format(group_name)) @@ -1180,6 +1188,7 @@ def get(self, request, group_name): "pas_config": pas_config, "is_cmp": is_cmp, "disallow_autoscaling": _disallow_autoscaling(curr_image), + "storage": get_messages(request) }) From 92fbbd4e8ab65c68888635d3a606f04b7e9deecd Mon Sep 17 00:00:00 2001 From: "Rodshell Fleurinord (he/him)" <69278532+rfleur01@users.noreply.github.com> Date: Tue, 13 Feb 2024 09:53:48 -0800 Subject: [PATCH 4/9] Updating messages banner to be more generic (#1451) * Revert "Add account id validator (#1348)" This reverts commit 42de3504dfc52b0586c4fb2ba76ec16ac97eb773. * Updating messages banner to be more generic --------- Co-authored-by: Rodshell Fleurinord --- deploy-board/deploy_board/templates/message_banner.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-board/deploy_board/templates/message_banner.tmpl b/deploy-board/deploy_board/templates/message_banner.tmpl index 68b32f7ccb..5452359dd0 100644 --- a/deploy-board/deploy_board/templates/message_banner.tmpl +++ b/deploy-board/deploy_board/templates/message_banner.tmpl @@ -8,7 +8,7 @@ {% else %} {% endif %} {% endfor %} From 120cd396fafcc2c779fdf73bb44508fa6d53c404 Mon Sep 17 00:00:00 2001 From: liyaqin1 <42289525+liyaqin1@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:00:19 -0800 Subject: [PATCH 5/9] Introduce Logs for Tracking Host Tags Addition in Database for Updator Identification (#1437) * Remove CMDB dependency in Teletraan worker * tmp save * emit metrics to monitor the tag creator * wait 2 minutes * rebase master * remove metrics - only use logs to track * fix a typo * remove sleep --- .../java/com/pinterest/deployservice/handler/GoalAnalyst.java | 4 ++-- .../java/com/pinterest/teletraan/worker/DeployTagWorker.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/GoalAnalyst.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/GoalAnalyst.java index 4f95dd4959..5ceacad141 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/GoalAnalyst.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/GoalAnalyst.java @@ -564,12 +564,12 @@ void process(String envId, EnvironBean env, PingReportBean report, AgentBean age hostTagBean.setEnv_id(envId); hostTagBean.setCreate_date(System.currentTimeMillis()); hostTagDAO.insertOrUpdate(hostTagBean); - LOG.debug("insert host_tags with env id {}, host id {}, tag name {}, tag value {}", envId, host_id, tagName, tagValue); + LOG.debug("Create host tags from Deployd: insert host_tags with env id {}, host id {}, tag name {}, tag value {}", envId, host_id, tagName, tagValue); } else if (tagValue.equals(hostTagBean.getTag_value()) == false) { hostTagBean.setTag_value(tagValue); hostTagBean.setCreate_date(System.currentTimeMillis()); hostTagDAO.insertOrUpdate(hostTagBean); - LOG.debug("update host_tags with env id {}, host id {}, tag name {}, tag value {}", envId, host_id, tagName, tagValue); + LOG.debug("Update host tags from Deployd: update host_tags with env id {}, host id {}, tag name {}, tag value {}", envId, host_id, tagName, tagValue); } } } diff --git a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployTagWorker.java b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployTagWorker.java index 74b1a791b8..8bf7bc7226 100644 --- a/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployTagWorker.java +++ b/deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/worker/DeployTagWorker.java @@ -116,6 +116,7 @@ private void processEachEnvironConstraint(DeployConstraintBean bean) throws Exce hostTagBean.setEnv_id(envId); hostTagBean.setCreate_date(System.currentTimeMillis()); statements.add(hostTagDAO.genInsertOrUpdate(hostTagBean)); + LOG.debug("Create host tags from CMDB: insert host_tags with env id {}, host id {}, tag name {}, tag value {}", envId, hostId, tagName, tagValue); } } } From c2d64a6269ad2ff29f11354a6b155c38c7a1add4 Mon Sep 17 00:00:00 2001 From: knguyen100000010 <63071572+knguyen100000010@users.noreply.github.com> Date: Fri, 16 Feb 2024 08:46:04 -0500 Subject: [PATCH 6/9] add az distribution pie chart (#1452) --- .../templates/groups/group_side_panel.html | 8 +++++ .../templates/groups/host_az_dist.tmpl | 31 +++++++++++++++++++ .../deploy_board/webapp/arcee_urls.py | 1 + .../deploy_board/webapp/group_view.py | 18 ++++++++++- 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 deploy-board/deploy_board/templates/groups/host_az_dist.tmpl diff --git a/deploy-board/deploy_board/templates/groups/group_side_panel.html b/deploy-board/deploy_board/templates/groups/group_side_panel.html index eea9c687c3..4e2eb9c66f 100644 --- a/deploy-board/deploy_board/templates/groups/group_side_panel.html +++ b/deploy-board/deploy_board/templates/groups/group_side_panel.html @@ -71,6 +71,14 @@

Group Config

{% endif %} +
+ + Availability Zone Distribution + +
+

Environments

diff --git a/deploy-board/deploy_board/templates/groups/host_az_dist.tmpl b/deploy-board/deploy_board/templates/groups/host_az_dist.tmpl new file mode 100644 index 0000000000..a5c4fe866f --- /dev/null +++ b/deploy-board/deploy_board/templates/groups/host_az_dist.tmpl @@ -0,0 +1,31 @@ +{% load utils %} + +{% block content %} +
+

Host Distribution Among Availability Zones

+ +
+ + + + +{% endblock %} \ No newline at end of file diff --git a/deploy-board/deploy_board/webapp/arcee_urls.py b/deploy-board/deploy_board/webapp/arcee_urls.py index 6e5536918b..e4d2b49b3d 100644 --- a/deploy-board/deploy_board/webapp/arcee_urls.py +++ b/deploy-board/deploy_board/webapp/arcee_urls.py @@ -67,6 +67,7 @@ url(r'^groups/names', group_view.get_group_names), url(r'^groups/search/(?P[a-zA-Z0-9\-_]+)/$', group_view.search_groups), url(r'^groups/search/$', group_view.group_landing), + url(r'^groups/(?P[a-zA-Z0-9\-_]+)/host-az-dist/$', group_view.get_host_az_dist), url(r'^groups/(?P[a-zA-Z0-9\-_]+)/$', group_view.GroupDetailView.as_view()), url(r'^groups/(?P[a-zA-Z0-9\-_]+)/config/$', group_view.GroupConfigView.as_view()), url(r'^groups/(?P[a-zA-Z0-9\-_]+)/envs/', group_view.get_envs), diff --git a/deploy-board/deploy_board/webapp/group_view.py b/deploy-board/deploy_board/webapp/group_view.py index 3e952c2f39..a284ee0dad 100644 --- a/deploy-board/deploy_board/webapp/group_view.py +++ b/deploy-board/deploy_board/webapp/group_view.py @@ -13,7 +13,7 @@ # limitations under the License. -from deploy_board.settings import IS_PINTEREST, PHOBOS_URL +from deploy_board.settings import CMDB_API_HOST, IS_PINTEREST, PHOBOS_URL from django.middleware.csrf import get_token from django.shortcuts import render, redirect from django.views.generic import View @@ -21,7 +21,9 @@ from django.http import HttpResponse from django.contrib import messages from django.contrib.messages import get_messages +from collections import Counter import json +import requests import logging import traceback from itertools import groupby @@ -1464,6 +1466,20 @@ def get_health_check_activities(request, group_name): }) +def get_host_az_dist(request, group_name): + host_az_dist = requests.post(url = CMDB_API_HOST+"/v2/query", json={ + "query": "tags.Autoscaling:{} AND state:running".format(group_name), + "fields": "location" + } + ) + + counter = Counter([x['location'] for x in host_az_dist.json()]) + + return render(request, 'groups/host_az_dist.tmpl', { + 'labels': list(counter.keys()), + 'data': list(counter.values()), + }) + def get_health_check_details(request, id): health_check = autoscaling_groups_helper.get_health_check(request, id) env = environs_helper.get(request, health_check.get('env_id')) From cc1d6fd3cb783e0be199e475e8459ca668838bd9 Mon Sep 17 00:00:00 2001 From: Ian Roquebert <72234714+gzpcho@users.noreply.github.com> Date: Fri, 16 Feb 2024 15:30:10 -0600 Subject: [PATCH 7/9] [deploy-agent] Return type hints in staging & types folders (#1405) * Add type hints --- deploy-agent/deployd/staging/stager.py | 11 ++++++----- deploy-agent/deployd/staging/transformer.py | 11 ++++++----- deploy-agent/deployd/types/build.py | 15 +++++++++------ deploy-agent/deployd/types/deploy_goal.py | 13 +++++++------ deploy-agent/deployd/types/ping_report.py | 4 ++-- deploy-agent/deployd/types/ping_response.py | 4 ++-- 6 files changed, 32 insertions(+), 26 deletions(-) diff --git a/deploy-agent/deployd/staging/stager.py b/deploy-agent/deployd/staging/stager.py index 14731ac9f0..935ff76576 100644 --- a/deploy-agent/deployd/staging/stager.py +++ b/deploy-agent/deployd/staging/stager.py @@ -20,6 +20,7 @@ import shutil import traceback import logging +from typing import Optional from deployd.common import LOG_FORMAT from deployd.common.caller import Caller @@ -35,7 +36,7 @@ class Stager(object): _script_dirname = "teletraan" _template_dirname = "teletraan_template" - def __init__(self, config, build, target, env_name, transformer=None): + def __init__(self, config, build, target, env_name, transformer=None) -> None: self._build_dir = config.get_builds_directory() self._user_role = config.get_user_role() agent_dir = config.get_agent_directory() @@ -45,7 +46,7 @@ def __init__(self, config, build, target, env_name, transformer=None): self._target = target self._env_name = env_name - def enable_package(self): + def enable_package(self) -> int: """Set the enabled build. """ old_build = self.get_enabled_build() @@ -90,7 +91,7 @@ def enable_package(self): finally: return status_code - def get_enabled_build(self): + def get_enabled_build(self) -> Optional[str]: """Figure out what build is enabled by looking at symlinks.""" if not os.path.exists(self._target): if (os.path.islink(self._target) and not @@ -110,7 +111,7 @@ def get_enabled_build(self): return symlink_target.rsplit("/", 1)[-1] - def transform_script(self): + def transform_script(self) -> None: script_dir = os.path.join(self._target, self._script_dirname) if not os.path.exists(script_dir): return @@ -125,7 +126,7 @@ def transform_script(self): script_dirname=self._script_dirname) -def main(): +def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-f', '--config-file', dest='config_file', default=None, help="the deploy agent conf file filename path. If none, " diff --git a/deploy-agent/deployd/staging/transformer.py b/deploy-agent/deployd/staging/transformer.py index 2d3a35a3ac..d020b8c693 100644 --- a/deploy-agent/deployd/staging/transformer.py +++ b/deploy-agent/deployd/staging/transformer.py @@ -17,6 +17,7 @@ from string import Template import re import logging +from typing import List from deployd import IS_PINTEREST log = logging.getLogger(__name__) @@ -29,12 +30,12 @@ class TeletraanTemplate(Template): class Transformer(object): - def __init__(self, agent_dir, env_name, dict_fn=None): + def __init__(self, agent_dir, env_name, dict_fn=None) -> None: self._agent_dir = agent_dir self._env_name = env_name self._load_config(dict_fn) - def _load_config(self, fn): + def _load_config(self, fn) -> None: if not fn: fn = os.path.join(self._agent_dir, "{}_SCRIPT_CONFIG".format(self._env_name)) @@ -45,10 +46,10 @@ def _load_config(self, fn): with open(fn, 'r') as f: self._dictionary = dict((n.strip('\"\n\' ') for n in line.split("=", 1)) for line in f) - def dict_size(self): + def dict_size(self) -> int: return len(self._dictionary) - def _translate(self, from_path, to_path): + def _translate(self, from_path, to_path) -> None: try: with open(from_path, 'r') as f: res = f.read() @@ -73,7 +74,7 @@ def _translate(self, from_path, to_path): log.error('Fail to translate script {}, stacktrace: {}'.format(from_path, traceback.format_exc())) - def transform_scripts(self, script_dir, template_dirname, script_dirname): + def transform_scripts(self, script_dir, template_dirname, script_dirname) -> List: scripts = [] suffix = ".tmpl" try: diff --git a/deploy-agent/deployd/types/build.py b/deploy-agent/deployd/types/build.py index db2753b536..e62f569718 100644 --- a/deploy-agent/deployd/types/build.py +++ b/deploy-agent/deployd/types/build.py @@ -13,9 +13,12 @@ # limitations under the License. +from typing import Tuple + + class Build(object): - def __init__(self, jsonValue=None): + def __init__(self, jsonValue=None) -> None: self.buildId = None self.buildName = None self.buildVersion = None @@ -43,7 +46,7 @@ def __init__(self, jsonValue=None): self.publishInfo = jsonValue.get('publishInfo') self.publishDate = jsonValue.get('publishDate') - def __key(self): + def __key(self) -> Tuple: return (self.buildId, self.buildName, self.buildVersion, @@ -57,20 +60,20 @@ def __key(self): self.publishInfo, self.publishDate) - def __hash__(self): + def __hash__(self) -> int: return hash(self.__key()) - def __eq__(self, other): + def __eq__(self, other) -> bool: """ compare Builds """ return isinstance(other, Build) \ and self.__key() == other.__key() - def __ne__(self, other): + def __ne__(self, other) -> bool: """ compare Builds """ return not (isinstance(other, Build) and self.__key() == other.__key()) - def __str__(self): + def __str__(self) -> str: return "Build(buildId={}, buildName={}, buildVersion={}, artifactUrl={}, scm={}, " \ "scmRepo={}, scmBranch={}, scmCommit={}, scmInfo={}, commitDate={}, publishInfo={}, " \ "publishDate={})".format(self.buildId, self.buildName, self.buildVersion, diff --git a/deploy-agent/deployd/types/deploy_goal.py b/deploy-agent/deployd/types/deploy_goal.py index 906f6d6b04..a4373c5870 100644 --- a/deploy-agent/deployd/types/deploy_goal.py +++ b/deploy-agent/deployd/types/deploy_goal.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Tuple from deployd.types.build import Build from deployd.types.deploy_stage import DeployStage class DeployGoal(object): - def __init__(self, jsonValue=None): + def __init__(self, jsonValue=None) -> None: self.deployId = None self.envId = None self.envName = None @@ -51,7 +52,7 @@ def __init__(self, jsonValue=None): self.firstDeploy = jsonValue.get('firstDeploy') self.isDocker = jsonValue.get('isDocker') - def __key(self): + def __key(self) -> Tuple: return (self.deployId, self.envId, self.envName, @@ -65,20 +66,20 @@ def __key(self): self.firstDeploy, self.isDocker) - def __hash__(self): + def __hash__(self) -> int: return hash(self.__key()) - def __eq__(self, other): + def __eq__(self, other) -> bool: """ compare DeployGoals """ return isinstance(other, DeployGoal) \ and self.__key() == other.__key() - def __ne__(self, other): + def __ne__(self, other) -> bool: """ compare DeployGoals """ return not (isinstance(other, DeployGoal) and self.__key() == other.__key()) - def __str__(self): + def __str__(self) -> str: return "DeployGoal(deployId={}, envId={}, envName={}, stageName={}, stageType={}, " \ "deployStage={}, build={}, deployAlias={}, agentConfig={}," \ "scriptVariables={}, firstDeploy={}, isDocker={})".format(self.deployId, self.envId, self.envName, diff --git a/deploy-agent/deployd/types/ping_report.py b/deploy-agent/deployd/types/ping_report.py index c73aa59c2a..601f0f1b94 100644 --- a/deploy-agent/deployd/types/ping_report.py +++ b/deploy-agent/deployd/types/ping_report.py @@ -18,7 +18,7 @@ class PingReport(object): - def __init__(self, jsonValue=None): + def __init__(self, jsonValue=None) -> None: self.deployId = None self.envId = None self.envName = None @@ -62,7 +62,7 @@ def __init__(self, jsonValue=None): self.redeploy = jsonValue.get('redeploy') self.wait = jsonValue.get('wait') - def __str__(self): + def __str__(self) -> str: return "PingReport(deployId={}, envId={}, deployStage={}, status={}, " \ "errorCode={}, errorMessage={}, failCount={}, extraInfo={}, " \ "deployAlias={}, containerHealthStatus={}, agentState={})".format(self.deployId, self.envId, self.deployStage, diff --git a/deploy-agent/deployd/types/ping_response.py b/deploy-agent/deployd/types/ping_response.py index 406922d002..312b5eac30 100644 --- a/deploy-agent/deployd/types/ping_response.py +++ b/deploy-agent/deployd/types/ping_response.py @@ -18,7 +18,7 @@ class PingResponse(object): - def __init__(self, jsonValue=None): + def __init__(self, jsonValue=None) -> None: self.opCode = OpCode.NOOP self.deployGoal = None @@ -32,5 +32,5 @@ def __init__(self, jsonValue=None): if jsonValue.get('deployGoal'): self.deployGoal = DeployGoal(jsonValue=jsonValue.get('deployGoal')) - def __str__(self): + def __str__(self) -> str: return "PingResponse(opCode={}, deployGoal={})".format(self.opCode, self.deployGoal) From da0be5cd4a185f1cc17854b98017a13569643cbf Mon Sep 17 00:00:00 2001 From: knguyen100000010 <63071572+knguyen100000010@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:28:57 -0500 Subject: [PATCH 8/9] Pie (#1457) * add more host dist stat * resume suspend process support * avoid hardcoding process name * remove Preview status --- .../clusters/cluster-replacements.tmpl | 2 +- .../templates/groups/asg_config.tmpl | 10 ++++ .../templates/groups/asg_processes.tmpl | 52 +++++++++++++++++++ .../templates/groups/host_az_dist.tmpl | 27 +++++++++- .../deploy_board/webapp/arcee_urls.py | 6 +++ .../deploy_board/webapp/group_view.py | 43 ++++++++++++++- .../helpers/autoscaling_groups_helper.py | 6 +++ 7 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 deploy-board/deploy_board/templates/groups/asg_processes.tmpl diff --git a/deploy-board/deploy_board/templates/clusters/cluster-replacements.tmpl b/deploy-board/deploy_board/templates/clusters/cluster-replacements.tmpl index 6082ab250f..74ec350e61 100644 --- a/deploy-board/deploy_board/templates/clusters/cluster-replacements.tmpl +++ b/deploy-board/deploy_board/templates/clusters/cluster-replacements.tmpl @@ -48,7 +48,7 @@
- Set up Auto Refresh (Preview) + Set up Auto Refresh
diff --git a/deploy-board/deploy_board/templates/groups/asg_config.tmpl b/deploy-board/deploy_board/templates/groups/asg_config.tmpl index 9656425aed..0522868bb7 100644 --- a/deploy-board/deploy_board/templates/groups/asg_config.tmpl +++ b/deploy-board/deploy_board/templates/groups/asg_config.tmpl @@ -70,6 +70,7 @@
+
@@ -102,6 +103,14 @@ function loadAsgScheduledActions() { }); } +function loadAsgSuspendedProcesses() { + $("#suspendedProcessesPid").addClass("panel panel-default"); + var asg_suspended_processes_url = "/groups/{{ group_name }}/autoscaling/get_suspended_processes/"; + $.get(asg_suspended_processes_url, function(response) { + $("#suspendedProcessesPid").html(response); + }); +} + function loadAsgPolicy() { $("#scalingPolicyPid").addClass("panel panel-default"); var asg_policy_url = "/groups/{{ group_name }}/autoscaling/get_asg_policy/"; @@ -121,6 +130,7 @@ function loadAsgAlarms() { function getAdvancedSetting() { loadAsgPolicy(); loadAsgScheduledActions(); + loadAsgSuspendedProcesses(); loadAsgAlarms(); } diff --git a/deploy-board/deploy_board/templates/groups/asg_processes.tmpl b/deploy-board/deploy_board/templates/groups/asg_processes.tmpl new file mode 100644 index 0000000000..30f4fe527a --- /dev/null +++ b/deploy-board/deploy_board/templates/groups/asg_processes.tmpl @@ -0,0 +1,52 @@ +{% load utils %} +{% include "panel_heading.tmpl" with panel_title="Suspended Processes" panel_body_id="asSuspendedProcessId" direction="down" %} +
+ +
+ + + + + + + {% for process in process_suspended_status %} + + + + + + {% endfor%} +
Process NameStateClick to ...
+ {{ process.name }} + + {% if process.suspended == True %}

Suspended

{% else %}

Active

{% endif %} + 		+ {% if process.suspended == True %} {% else %} {% endif %} +
+
+ + +
diff --git a/deploy-board/deploy_board/templates/groups/host_az_dist.tmpl b/deploy-board/deploy_board/templates/groups/host_az_dist.tmpl index a5c4fe866f..b554fdc613 100644 --- a/deploy-board/deploy_board/templates/groups/host_az_dist.tmpl +++ b/deploy-board/deploy_board/templates/groups/host_az_dist.tmpl @@ -1,11 +1,36 @@ {% load utils %} {% block content %} -
+

Host Distribution Among Availability Zones

+
+ + + + + + + {% for l, d, p in label_data_percentage %} + + + + + + {% endfor%} +
Availability ZoneHost CountPercentage
+ {{ l }} + + {{ d }} + + {{ p }}% +
+

==============================

+ Total: {{ total }} hosts +
+