Merge pull request #7643 from AlanCoding/per_update_cache

Implement per-update cache for roles and collections

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/main/models/projects.py

@@ -194,6 +194,11 @@ def get_project_path(self, check_if_exists=True):
             if not check_if_exists or os.path.exists(smart_str(proj_path)):
                 return proj_path
 
+    def get_cache_path(self):
+        local_path = os.path.basename(self.local_path)
+        if local_path:
+            return os.path.join(settings.PROJECTS_ROOT, '.__awx_cache', local_path)
+
     @property
     def playbooks(self):
         results = []
@@ -418,6 +423,10 @@ def needs_update_on_launch(self):
                 return True
         return False
 
+    @property
+    def cache_id(self):
+        return str(self.last_job_id)
+
     @property
     def notification_templates(self):
         base_notification_templates = NotificationTemplate.objects
@@ -455,11 +464,12 @@ def _get_related_jobs(self):
         )
 
     def delete(self, *args, **kwargs):
-        path_to_delete = self.get_project_path(check_if_exists=False)
+        paths_to_delete = (self.get_project_path(check_if_exists=False), self.get_cache_path())
         r = super(Project, self).delete(*args, **kwargs)
-        if self.scm_type and path_to_delete:  # non-manual, concrete path
-            from awx.main.tasks import delete_project_files
-            delete_project_files.delay(path_to_delete)
+        for path_to_delete in paths_to_delete:
+            if self.scm_type and path_to_delete:  # non-manual, concrete path
+                from awx.main.tasks import delete_project_files
+                delete_project_files.delay(path_to_delete)
         return r
 
 
@@ -554,6 +564,19 @@ def result_stdout(self):
     def result_stdout_raw(self):
         return self._result_stdout_raw(redact_sensitive=True)
 
+    @property
+    def branch_override(self):
+        """Whether a branch other than the project default is used."""
+        if not self.project:
+            return True
+        return bool(self.scm_branch and self.scm_branch != self.project.scm_branch)
+
+    @property
+    def cache_id(self):
+        if self.branch_override or self.job_type == 'check' or (not self.project):
+            return str(self.id)
+        return self.project.cache_id
+
     def result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True):
         return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive=redact_sensitive)
 
@@ -597,10 +620,7 @@ def preferred_instance_groups(self):
     def save(self, *args, **kwargs):
         added_update_fields = []
         if not self.job_tags:
-            job_tags = ['update_{}'.format(self.scm_type)]
-            if self.job_type == 'run':
-                job_tags.append('install_roles')
-                job_tags.append('install_collections')
+            job_tags = ['update_{}'.format(self.scm_type), 'install_roles', 'install_collections']
             self.job_tags = ','.join(job_tags)
             added_update_fields.append('job_tags')
         if self.scm_delete_on_update and 'delete' not in self.job_tags and self.job_type == 'check':

awx/main/tests/functional/api/test_project.py

@@ -54,7 +54,9 @@ def test_no_changing_overwrite_behavior_if_used(post, patch, organization, admin
         data={
             'name': 'fooo',
             'organization': organization.id,
-            'allow_override': True
+            'allow_override': True,
+            'scm_type': 'git',
+            'scm_url': 'https://github.com/ansible/test-playbooks.git'
         },
         user=admin_user,
         expect=201
@@ -83,7 +85,9 @@ def test_changing_overwrite_behavior_okay_if_not_used(post, patch, organization,
         data={
             'name': 'fooo',
             'organization': organization.id,
-            'allow_override': True
+            'allow_override': True,
+            'scm_type': 'git',
+            'scm_url': 'https://github.com/ansible/test-playbooks.git'
         },
         user=admin_user,
         expect=201

awx/main/tests/functional/conftest.py

@@ -145,7 +145,6 @@ def project(instance, organization):
                                  description="test-proj-desc",
                                  organization=organization,
                                  playbook_files=['helloworld.yml', 'alt-helloworld.yml'],
-                                 local_path='_92__test_proj',
                                  scm_revision='1234567890123456789012345678901234567890',
                                  scm_url='localhost',
                                  scm_type='git'

awx/main/tests/functional/models/test_inventory.py

@@ -169,7 +169,8 @@ def test_source_location(self, scm_inventory_source):
         inventory_update = InventoryUpdate(
             inventory_source=scm_inventory_source,
             source_path=scm_inventory_source.source_path)
-        assert inventory_update.get_actual_source_path().endswith('_92__test_proj/inventory_file')
+        p = scm_inventory_source.source_project
+        assert inventory_update.get_actual_source_path().endswith(f'_{p.id}__test_proj/inventory_file')
 
     def test_no_unwanted_updates(self, scm_inventory_source):
         # Changing the non-sensitive fields should not trigger update

awx/main/tests/functional/models/test_project.py

@@ -34,6 +34,18 @@ def test_sensitive_change_triggers_update(project):
     mock_update.assert_called_once_with()
 
 
+@pytest.mark.django_db
+def test_local_path_autoset(organization):
+    with mock.patch.object(Project, "update"):
+        p = Project.objects.create(
+            name="test-proj",
+            organization=organization,
+            scm_url='localhost',
+            scm_type='git'
+        )
+    assert p.local_path == f'_{p.id}__test_proj'
+
+
 @pytest.mark.django_db
 def test_foreign_key_change_changes_modified_by(project, organization):
     assert project._get_fields_snapshot()['organization_id'] == organization.id

awx/main/tests/functional/test_projects.py

@@ -29,8 +29,8 @@ def team_project_list(organization_factory):
 @pytest.mark.django_db
 def test_get_project_path(project):
     # Test combining projects root with project local path
-    with mock.patch('awx.main.models.projects.settings.PROJECTS_ROOT', '/var/lib/awx'):
-        assert project.get_project_path(check_if_exists=False) == '/var/lib/awx/_92__test_proj'
+    with mock.patch('awx.main.models.projects.settings.PROJECTS_ROOT', '/var/lib/foo'):
+        assert project.get_project_path(check_if_exists=False) == f'/var/lib/foo/_{project.id}__test_proj'
 
 
 @pytest.mark.django_db

awx/main/tests/functional/test_tasks.py

@@ -30,7 +30,7 @@ class TestDependentInventoryUpdate:
     def test_dependent_inventory_updates_is_called(self, scm_inventory_source, scm_revision_file):
         task = RunProjectUpdate()
         task.revision_path = scm_revision_file
-        proj_update = ProjectUpdate.objects.create(project=scm_inventory_source.source_project)
+        proj_update = scm_inventory_source.source_project.create_project_update()
         with mock.patch.object(RunProjectUpdate, '_update_dependent_inventories') as inv_update_mck:
             with mock.patch.object(RunProjectUpdate, 'release_lock'):
                 task.post_run_hook(proj_update, 'successful')
@@ -39,7 +39,7 @@ def test_dependent_inventory_updates_is_called(self, scm_inventory_source, scm_r
     def test_no_unwanted_dependent_inventory_updates(self, project, scm_revision_file):
         task = RunProjectUpdate()
         task.revision_path = scm_revision_file
-        proj_update = ProjectUpdate.objects.create(project=project)
+        proj_update = project.create_project_update()
         with mock.patch.object(RunProjectUpdate, '_update_dependent_inventories') as inv_update_mck:
             with mock.patch.object(RunProjectUpdate, 'release_lock'):
                 task.post_run_hook(proj_update, 'successful')

awx/main/tests/unit/test_tasks.py

@@ -61,7 +61,10 @@ def patch_Job():
 
 @pytest.fixture
 def job():
-    return Job(pk=1, id=1, project=Project(), inventory=Inventory(), job_template=JobTemplate(id=1, name='foo'))
+    return Job(
+        pk=1, id=1,
+        project=Project(local_path='/projects/_23_foo'),
+        inventory=Inventory(), job_template=JobTemplate(id=1, name='foo'))
 
 
 @pytest.fixture
@@ -406,7 +409,9 @@ def test_overwritten_jt_extra_vars(self, job, private_data_dir):
 class TestGenericRun():
 
     def test_generic_failure(self, patch_Job):
-        job = Job(status='running', inventory=Inventory(), project=Project())
+        job = Job(
+            status='running', inventory=Inventory(),
+            project=Project(local_path='/projects/_23_foo'))
         job.websocket_emit_status = mock.Mock()
 
         task = tasks.RunJob()

awx/playbooks/project_update.yml

@@ -1,6 +1,9 @@
 ---
 # The following variables will be set by the runner of this playbook:
-# project_path: PROJECTS_DIR/_local_path_
+# projects_root: Global location for caching project checkouts and roles and collections
+#     should not have trailing slash on end
+# local_path: Path within projects_root to use for this project
+# project_path: A simple join of projects_root/local_path folders
 # scm_url: https://server/repo
 # insights_url: Insights service URL (from configuration)
 # scm_branch: branch/tag/revision (HEAD if unset)
@@ -11,8 +14,6 @@
 # scm_refspec: a refspec to fetch in addition to obtaining version
 # roles_enabled: Value of the global setting to enable roles downloading
 # collections_enabled: Value of the global setting to enable collections downloading
-# roles_destination: Path to save roles from galaxy to
-# collections_destination: Path to save collections from galaxy to
 # awx_version: Current running version of the awx or tower as a string
 # awx_license_type: "open" for AWX; else presume Tower
 
@@ -122,7 +123,10 @@
           register: doesRequirementsExist
 
         - name: fetch galaxy roles from requirements.yml
-          command: ansible-galaxy role install -r roles/requirements.yml -p {{roles_destination|quote}}{{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
+          command: >
+            ansible-galaxy role install -r roles/requirements.yml
+            --roles-path {{projects_root}}/.__awx_cache/{{local_path}}/stage/requirements_roles
+            {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
           args:
             chdir: "{{project_path|quote}}"
           register: galaxy_result
@@ -143,15 +147,18 @@
           register: doesCollectionRequirementsExist
 
         - name: fetch galaxy collections from collections/requirements.yml
-          command: ansible-galaxy collection install -r collections/requirements.yml -p {{collections_destination|quote}}{{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
+          command: >
+            ansible-galaxy collection install -r collections/requirements.yml
+            --collections-path {{projects_root}}/.__awx_cache/{{local_path}}/stage/requirements_collections
+            {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
           args:
             chdir: "{{project_path|quote}}"
           register: galaxy_collection_result
           when: doesCollectionRequirementsExist.stat.exists
           changed_when: "'Installing ' in galaxy_collection_result.stdout"
           environment:
             ANSIBLE_FORCE_COLOR: false
-            ANSIBLE_COLLECTIONS_PATHS: "{{ collections_destination }}"
+            ANSIBLE_COLLECTIONS_PATHS: "{{projects_root}}/.__awx_cache/{{local_path}}/stage/requirements_collections"
             GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no"
 
       when:

docs/collections.md

@@ -4,23 +4,26 @@ AWX supports the use of Ansible Collections. This section will give ways to use
 
 ### Project Collections Requirements
 
-If you specify a Collections requirements file in SCM at `collections/requirements.yml`,
-then AWX will install Collections from that file in the implicit project sync
-before a job run. The invocation looks like:
+If you specify a collections requirements file in SCM at `collections/requirements.yml`,
+then AWX will install collections from that file to a special cache folder in project updates.
+Before a job runs, the roles and/or collections will be copied from the special
+cache folder to the job temporary folder.
+
+The invocation looks like:
 
 ```
-ansible-galaxy collection install -r requirements.yml -p <job tmp location>/requirements_collections
+ansible-galaxy collection install -r requirements.yml -p <project cache location>/requirements_collections
 ```
 
-Example of the resultant `tmp` directory where job is running:
+Example of the resultant job `tmp` directory where job is running:
 
 ```
 ├── project
 │   ├── ansible.cfg
 │   └── debug.yml
 ├── requirements_collections
 │   └── ansible_collections
-│       └── username
+│       └── collection_namespace
 │           └── collection_name
 │               ├── FILES.json
 │               ├── MANIFEST.json
@@ -53,6 +56,33 @@ Example of the resultant `tmp` directory where job is running:
 
 ```
 
+### Cache Folder Mechanics
+
+Every time a project is updated as a "check" job
+(via `/api/v2/projects/N/update/` or by a schedule, workflow, etc.),
+the roles and collections are downloaded and saved to the project's content cache.
+In other words, the cache is invalidated every time a project is updated.
+That means that the `ansible-galaxy` commands are ran to download content
+even if the project revision does not change in the course of the update.
+
+Project updates all initially target a staging directory at a path like:
+
+```
+/var/lib/awx/projects/.__awx_cache/_42__project_name/stage
+```
+
+After the update finishes, the task logic will decide what id to associate
+with the content downloaded.
+Then the folder will be renamed from "stage" to the cache id.
+For instance, if the cache id is determined to be 63:
+
+```
+/var/lib/awx/projects/.__awx_cache/_42__project_name/63
+```
+
+The cache may be updated by project syncs (the "run" type) which happen before
+job runs. It will populate the cache id set by the last "check" type update.
+
 ### Galaxy Server Selection
 
 Ansible core default settings will download collections from the public