diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 0321cf910fba..c3dddc9d9a22 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -679,9 +679,12 @@ class BaseTask(object): model = None event_model = None abstract = True - cleanup_paths = [] proot_show_paths = [] + def __init__(self, *args, **kwargs): + super(BaseTask, self).__init__(*args, **kwargs) + self.cleanup_paths = [] + def update_model(self, pk, _attempt=0, **updates): """Reload the model instance from the database and update the given fields. @@ -992,7 +995,7 @@ def create_expect_passwords_data_struct(self, password_prompts, passwords): expect_passwords[k] = passwords.get(v, '') or '' return expect_passwords - def pre_run_hook(self, instance): + def pre_run_hook(self, instance, private_data_dir): ''' Hook for any steps to run before the job/task starts ''' @@ -1118,7 +1121,8 @@ def run(self, pk, **kwargs): try: isolated = self.instance.is_isolated() - self.pre_run_hook(self.instance) + private_data_dir = self.build_private_data_dir(self.instance) + self.pre_run_hook(self.instance, private_data_dir) if self.instance.cancel_flag: self.instance = self.update_model(self.instance.pk, status='canceled') if self.instance.status != 'running': @@ -1134,7 +1138,6 @@ def run(self, pk, **kwargs): # store a record of the venv used at runtime if hasattr(self.instance, 'custom_virtualenv'): self.update_model(pk, custom_virtualenv=getattr(self.instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH)) - private_data_dir = self.build_private_data_dir(self.instance) # Fetch "cached" fact data from prior runs and put on the disk # where ansible expects to find it @@ -1217,9 +1220,6 @@ def run(self, pk, **kwargs): module_args = ansible_runner.utils.args2cmdline( params.get('module_args'), ) - else: - # otherwise, it's a playbook, so copy the project dir - copy_tree(cwd, os.path.join(private_data_dir, 'project')) shutil.move( params.pop('inventory'), os.path.join(private_data_dir, 'inventory') @@ -1489,15 +1489,10 @@ def build_args(self, job, private_data_dir, passwords): return args def build_cwd(self, job, private_data_dir): - cwd = job.project.get_project_path() - if not cwd: - root = settings.PROJECTS_ROOT - raise RuntimeError('project local_path %s cannot be found in %s' % - (job.project.local_path, root)) - return cwd + return os.path.join(private_data_dir, 'project') def build_playbook_path_relative_to_cwd(self, job, private_data_dir): - return os.path.join(job.playbook) + return job.playbook def build_extra_vars_file(self, job, private_data_dir): # Define special extra_vars for AWX, combine with job.extra_vars. @@ -1543,11 +1538,12 @@ def should_use_proot(self, job): ''' return getattr(settings, 'AWX_PROOT_ENABLED', False) - def pre_run_hook(self, job): + def pre_run_hook(self, job, private_data_dir): if job.inventory is None: error = _('Job could not start because it does not have a valid inventory.') self.update_model(job.pk, status='failed', job_explanation=error) raise RuntimeError(error) + galaxy_install_path = None if job.project and job.project.scm_type: pu_ig = job.instance_group pu_en = job.execution_node @@ -1572,9 +1568,13 @@ def pre_run_hook(self, job): # cancel() call on the job can cancel the project update job = self.update_model(job.pk, project_update=local_project_sync) + # Save the roles from galaxy to a temporary directory to be moved later + # at this point, the project folder has not yet been coppied into the temporary directory + galaxy_install_path = tempfile.mkdtemp(prefix='tmp_roles_', dir=private_data_dir) + os.chmod(galaxy_install_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) project_update_task = local_project_sync._get_task_class() try: - project_update_task().run(local_project_sync.id) + project_update_task(roles_destination=galaxy_install_path).run(local_project_sync.id) job = self.update_model(job.pk, scm_revision=job.project.scm_revision) except Exception: local_project_sync.refresh_from_db() @@ -1584,6 +1584,21 @@ def pre_run_hook(self, job): ('project_update', local_project_sync.name, local_project_sync.id))) raise + # copy the project and roles directory + project_path = job.project.get_project_path(check_if_exists=False) + self.copy_folders(project_path, galaxy_install_path, private_data_dir) + + def copy_folders(self, project_path, galaxy_install_path, private_data_dir): + if project_path is None: + raise RuntimeError('project does not supply a valid path') + elif not os.path.exists(project_path): + raise RuntimeError('project path %s cannot be found' % project_path) + runner_project_folder = os.path.join(private_data_dir, 'project') + copy_tree(project_path, runner_project_folder) + if galaxy_install_path: + galaxy_run_path = os.path.join(private_data_dir, 'project', 'roles') + copy_tree(galaxy_install_path, galaxy_run_path) + def final_run_hook(self, job, status, private_data_dir, fact_modification_times, isolated_manager_instance=None): super(RunJob, self).final_run_hook(job, status, private_data_dir, fact_modification_times) if not private_data_dir: @@ -1617,6 +1632,10 @@ class RunProjectUpdate(BaseTask): def proot_show_paths(self): return [settings.PROJECTS_ROOT] + def __init__(self, *args, roles_destination=None, **kwargs): + super(RunProjectUpdate, self).__init__(*args, **kwargs) + self.roles_destination = roles_destination + def build_private_data(self, project_update, private_data_dir): ''' Return SSH private key data needed for this project update. @@ -1750,8 +1769,10 @@ def build_extra_vars_file(self, project_update, private_data_dir): 'scm_full_checkout': True if project_update.job_type == 'run' else False, 'scm_revision_output': self.revision_path, 'scm_revision': project_update.project.scm_revision, - 'roles_enabled': getattr(settings, 'AWX_ROLES_ENABLED', True) + 'roles_enabled': getattr(settings, 'AWX_ROLES_ENABLED', True) if project_update.job_type != 'check' else False }) + if self.roles_destination: + extra_vars['roles_destination'] = self.roles_destination self._write_extra_vars_file(private_data_dir, extra_vars) def build_cwd(self, project_update, private_data_dir): @@ -1872,7 +1893,7 @@ def acquire_lock(self, instance, blocking=True): '{} spent {} waiting to acquire lock for local source tree ' 'for path {}.'.format(instance.log_format, waiting_time, lock_path)) - def pre_run_hook(self, instance): + def pre_run_hook(self, instance, private_data_dir): # re-create root project folder if a natural disaster has destroyed it if not os.path.exists(settings.PROJECTS_ROOT): os.mkdir(settings.PROJECTS_ROOT) @@ -2111,11 +2132,12 @@ def build_credentials_list(self, inventory_update): # All credentials not used by inventory source injector return inventory_update.get_extra_credentials() - def pre_run_hook(self, inventory_update): + def pre_run_hook(self, inventory_update, private_data_dir): source_project = None if inventory_update.inventory_source: source_project = inventory_update.inventory_source.source_project if (inventory_update.source=='scm' and inventory_update.launch_type!='scm' and source_project): + # In project sync, pulling galaxy roles is not needed local_project_sync = source_project.create_project_update( _eager_fields=dict( launch_type="sync", diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 143e461daf3e..f2e96eb0ea40 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -359,12 +359,13 @@ def test_overwritten_jt_extra_vars(self, job, private_data_dir): class TestGenericRun(): def test_generic_failure(self, patch_Job): - job = Job(status='running', inventory=Inventory()) + job = Job(status='running', inventory=Inventory(), project=Project()) job.websocket_emit_status = mock.Mock() task = tasks.RunJob() task.update_model = mock.Mock(return_value=job) task.build_private_data_files = mock.Mock(side_effect=OSError()) + task.copy_folders = mock.Mock() with pytest.raises(Exception): task.run(1) @@ -382,6 +383,7 @@ def test_cancel_flag(self, job, update_model_wrapper): task = tasks.RunJob() task.update_model = mock.Mock(wraps=update_model_wrapper) task.build_private_data_files = mock.Mock() + task.copy_folders = mock.Mock() with pytest.raises(Exception): task.run(1) diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index 14c46ed1adb2..2fbdb2de5371 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -14,6 +14,7 @@ # scm_revision: current revision in tower # scm_revision_output: where to store gathered revision (temporary file) # roles_enabled: Allow us to pull roles from a requirements.yml file +# roles_destination: Path to save roles from galaxy to # awx_version: Current running version of the awx or tower as a string # awx_license_type: "open" for AWX; else presume Tower @@ -148,18 +149,12 @@ register: doesRequirementsExist - name: fetch galaxy roles from requirements.yml - command: ansible-galaxy install -r requirements.yml -p {{project_path|quote}}/roles/ + command: ansible-galaxy install -r requirements.yml -p {{roles_destination|quote}} args: chdir: "{{project_path|quote}}/roles" register: galaxy_result - when: doesRequirementsExist.stat.exists and (scm_version is undefined or (git_result is defined and git_result['before'] == git_result['after'])) + when: doesRequirementsExist.stat.exists changed_when: "'was installed successfully' in galaxy_result.stdout" - - name: fetch galaxy roles from requirements.yml (forced update) - command: ansible-galaxy install -r requirements.yml -p {{project_path|quote}}/roles/ --force - args: - chdir: "{{project_path|quote}}/roles" - when: doesRequirementsExist.stat.exists and galaxy_result is skipped - when: roles_enabled|bool delegate_to: localhost