diff --git a/tools/distribution/dogfood.py b/tools/distribution/dogfood.py index 5144cd1017..53e903a061 100755 --- a/tools/distribution/dogfood.py +++ b/tools/distribution/dogfood.py @@ -11,7 +11,7 @@ import os import traceback from tempfile import TemporaryDirectory -from src.dogfood.package_resolved import PackageResolvedFile +from src.dogfood.package_resolved import PackageResolvedFile, PackageID from src.dogfood.dogfooded_commit import DogfoodedCommit from src.dogfood.repository import Repository from src.utils import remember_cwd @@ -28,6 +28,12 @@ def dogfood(dry_run: bool, repository_url: str, repository_name: str, repository os.system(f'swift package --package-path {dd_sdk_package_path} resolve') dd_sdk_ios_package = PackageResolvedFile(path=f'{dd_sdk_package_path}/Package.resolved') + if dd_sdk_ios_package.version != 1: + raise Exception( + f'`dogfood.py` expects the `package.resolved` in `dd-sdk-ios` to use version 1 ' + + f'but version {dd_sdk_ios_package.version} was detected. Update `dogfood.py` to use this version.' + ) + # Clone dependant repo to temporary location and update its `Package.resolved` (one or many) so it points # to the current `dd-sdk-ios` commit. After that, push changes to dependant repo and create dogfooding PR. with TemporaryDirectory() as temp_dir: @@ -46,26 +52,26 @@ def dogfood(dry_run: bool, repository_url: str, repository_name: str, repository # Update version of `dd-sdk-ios`: for package in packages: package.update_dependency( - package_name='DatadogSDK', + package_id=PackageID(v1='DatadogSDK', v2='dd-sdk-ios'), new_branch='dogfooding', new_revision=dd_sdk_ios_commit.hash, new_version=None ) # Add or update `dd-sdk-ios` dependencies - for dependency_name in dd_sdk_ios_package.read_dependency_names(): - dependency = dd_sdk_ios_package.read_dependency(package_name=dependency_name) + for dependency_id in dd_sdk_ios_package.read_dependency_ids(): + dependency = dd_sdk_ios_package.read_dependency(package_id=dependency_id) - if package.has_dependency(package_name=dependency_name): + if package.has_dependency(package_id=dependency_id): package.update_dependency( - package_name=dependency_name, + package_id=dependency_id, new_branch=dependency['state']['branch'], new_revision=dependency['state']['revision'], new_version=dependency['state']['version'], ) else: package.add_dependency( - package_name=dependency_name, + package_id=dependency_id, repository_url=dependency['repositoryURL'], branch=dependency['state']['branch'], revision=dependency['state']['revision'], diff --git a/tools/distribution/src/dogfood/package_resolved.py b/tools/distribution/src/dogfood/package_resolved.py index 3afd5ba3ef..c034ab4aab 100644 --- a/tools/distribution/src/dogfood/package_resolved.py +++ b/tools/distribution/src/dogfood/package_resolved.py @@ -5,53 +5,167 @@ # ----------------------------------------------------------- import json +from dataclasses import dataclass from copy import deepcopy +from typing import Optional -class PackageResolvedFile: +@dataclass() +class PackageID: + """ + Identifies package in `package.resolved` file. + It supports `version: 1` and `version: 2` of `package.resolved` format: + - v1 uses package name (e.g. `DatadogSDK`) as package identifier + - v2 uses package identity (e.g. `dd-sdk-ios`) as package identifier + - v2 is not backward compatible with v1 - the v1 package name cannot be read from v2's `package.resolved` + - v1 is forward compatible with v2 - the v2 package identity can be read from `repositoryURL` in v1's `package.resolved` + """ + v1: Optional[str] # can be `None` if read from v2's `package.resolved` + v2: str + + +def v2_package_id_from_repository_url(repository_url: str) -> str: + """Reads v2 package id from repository URL.""" + components = repository_url.split('/') # e.g. ['https:/', '', 'github.com', 'A-org', 'abc.git'] + return components[-1].split('.')[0] + + +class PackageResolvedContent: + """An interface for manipulating `package.resolved` content.""" + + def has_dependency(self, package_id: PackageID) -> bool: + """Checks if dependency with given ID exists.""" + pass + + def update_dependency(self, package_id: PackageID, new_branch: Optional[str], new_revision: str, new_version: Optional[str]): + """ + Updates dependency resolution values. + :param package_id: identifies dependency to update + :param new_branch: the new branch name (pass `None` for `null`) + :param new_revision: the new revision (pass `None` for `null`) + :param new_version: the new version name (pass `None` for `null`) + :return: + """ + pass + + def add_dependency(self, package_id: PackageID, repository_url: str, branch: Optional[str], revision: str, version: Optional[str]): + """ + Adds new dependency resolution. + """ + pass + + def read_dependency_ids(self) -> [PackageID]: + """ + Returns package IDs for all dependencies. + :return: list of package IDs (PackageIDs) + """ + pass + + def read_dependency(self, package_id: PackageID) -> dict: + """ + Returns resolution info for given dependency. + :param package_id: the `PackageID` of dependency + :return: the `pin` object from `Package.resolved` + """ + pass + + +class PackageResolvedFile(PackageResolvedContent): """ Abstracts operations on `Package.resolved` file. """ - __SUPPORTED_PACKAGE_RESOLVED_VERSION = 1 + version: int + wrapped: PackageResolvedContent def __init__(self, path: str): print(f'⚙️ Opening {path}') self.path = path with open(path, 'r') as file: self.packages = json.load(file) - version = self.packages['version'] - if version != self.__SUPPORTED_PACKAGE_RESOLVED_VERSION: + self.version = self.packages['version'] + if self.version == 1: + self.wrapped = PackageResolvedContentV1(self.path, self.packages) + elif self.version == 2: + self.wrapped = PackageResolvedContentV2(self.path, self.packages) + else: raise Exception( - f'{path} uses version {version} but `package_resolved.py` supports ' + - f'version {self.__SUPPORTED_PACKAGE_RESOLVED_VERSION}. Update `package_resolved.py` to new format.' + f'{path} uses version {self.version} but `PackageResolvedFile` only supports ' + + f'versions `1` and `2`. Update `PackageResolvedFile` to support new version.' ) - def has_dependency(self, package_name: str): - pins = self.packages['object']['pins'] - return package_name in [p['package'] for p in pins] + def save(self): + """ + Saves changes to initial `path`. + """ + print(f'⚙️ Saving {self.path}') + with open(self.path, 'w') as file: + json.dump( + self.packages, + fp=file, + indent=2, # preserve `swift package` indentation + separators=(',', ': ' if self.version == 1 else ' : '), # v1: `"key": "value"`, v2: `"key" : "value"` + sort_keys=True # preserve `swift package` packages sorting + ) + file.write('\n') # add new line to the EOF - def update_dependency(self, package_name: str, new_branch: str, new_revision: str, new_version): + def print(self): """ - Updates dependency resolution values. - :param package_name: the name of the package to update - :param new_branch: the new branch name (pass `None` for `null`) - :param new_revision: the new revision (pass `None` for `null`) - :param new_version: the new version name (pass `None` for `null`) - :return: + Prints the content of this file. """ - package = self.__get_package(package_name=package_name) - - # Individual package pin looks this: - # { - # "package": "DatadogSDK", - # "repositoryURL": "https://github.com/DataDog/dd-sdk-ios", - # "state": { - # "branch": "dogfooding", - # "revision": "4e93a8f1f662d9126074a0f355b4b6d20f9f30a7", - # "version": null - # } - # } + with open(self.path, 'r') as file: + print(f'⚙️ Content of {file.name}:') + print(file.read()) + + def has_dependency(self, package_id: PackageID) -> bool: + return self.wrapped.has_dependency(package_id) + + def update_dependency(self, package_id: PackageID, new_branch: Optional[str], new_revision: str, new_version: Optional[str]): + self.wrapped.update_dependency(package_id, new_branch, new_revision, new_version) + + def add_dependency(self, package_id: PackageID, repository_url: str, branch: Optional[str], revision: str, version: Optional[str]): + self.wrapped.add_dependency(package_id, repository_url, branch, revision, version) + + def read_dependency_ids(self) -> [PackageID]: + return self.wrapped.read_dependency_ids() + + def read_dependency(self, package_id: PackageID) -> dict: + return self.wrapped.read_dependency(package_id) + + +class PackageResolvedContentV1(PackageResolvedContent): + """ + Example of `package.resolved` in version `1` looks this:: + + { + "object": { + "pins": [ + { + "package": "DatadogSDK", + "repositoryURL": "https://github.com/DataDog/dd-sdk-ios", + "state": { + "branch": "dogfooding", + "revision": "4e93a8f1f662d9126074a0f355b4b6d20f9f30a7", + "version": null + } + }, + ... + ] + }, + "version": 1 + } + """ + + def __init__(self, path: str, json_content: dict): + self.path = path + self.packages = json_content + + def has_dependency(self, package_id: PackageID): + pins = self.packages['object']['pins'] + return package_id.v1 in [p['package'] for p in pins] + + def update_dependency(self, package_id: PackageID, new_branch: Optional[str], new_revision: str, new_version: Optional[str]): + package = self.__get_package(package_id=package_id) old_state = deepcopy(package['state']) @@ -64,37 +178,22 @@ def update_dependency(self, package_name: str, new_branch: str, new_revision: st diff = old_state.items() ^ new_state.items() if len(diff) > 0: - print(f'✏️️ Updated "{package_name}" in {self.path}:') + print(f'✏️️ Updated "{package_id.v1}" in {self.path}:') print(f' → old: {old_state}') print(f' → new: {new_state}') else: - print(f'✏️️ "{package_name}" is up-to-date in {self.path}') - - def add_dependency(self, package_name: str, repository_url: str, branch: str, revision: str, version): - """ - Inserts new dependency resolution to this `Package.resolved`. - """ + print(f'✏️️ "{package_id.v1}" is up-to-date in {self.path}') + def add_dependency(self, package_id: PackageID, repository_url: str, branch: Optional[str], revision: str, version: Optional[str]): pins = self.packages['object']['pins'] # Find the index in `pins` array where the new dependency should be inserted. # The `pins` array seems to follow the alphabetical order, but not always # - I've seen `Package.resolved` where some dependencies were misplaced. - index = next((i for i in range(len(pins)) if pins[i]['package'].lower() > package_name.lower()), len(pins)) - - # Individual package pin looks this: - # { - # "package": "DatadogSDK", - # "repositoryURL": "https://github.com/DataDog/dd-sdk-ios", - # "state": { - # "branch": "dogfooding", - # "revision": "4e93a8f1f662d9126074a0f355b4b6d20f9f30a7", - # "version": null - # } - # } + index = next((i for i in range(len(pins)) if pins[i]['package'].lower() > package_id.v1.lower()), len(pins)) new_pin = { - 'package': package_name, + 'package': package_id.v1, 'repositoryURL': repository_url, 'state': { 'branch': branch, @@ -105,59 +204,143 @@ def add_dependency(self, package_name: str, repository_url: str, branch: str, re pins.insert(index, new_pin) - print(f'✏️️ Added "{package_name}" at index {index} in {self.path}:') + print(f'✏️️ Added "{package_id.v1}" at index {index} in {self.path}:') print(f' → branch: {branch}') print(f' → revision: {revision}') print(f' → version: {version}') - def read_dependency_names(self): - """ - Returns package names for all dependencies in this `Package.resolved` file. - :return: list of package names (strings) - """ + def read_dependency_ids(self): pins = self.packages['object']['pins'] - package_names = [pin['package'] for pin in pins] - return package_names + package_ids = [PackageID(v1=pin['package'], v2=v2_package_id_from_repository_url(pin['repositoryURL'])) for pin in pins] + return package_ids - def read_dependency(self, package_name): - """ - Returns resolution info for given dependency. - :param package_name: the name of dependency - :return: the `pin` object from `Package.resolved` - """ - package = self.__get_package(package_name=package_name) + def read_dependency(self, package_id: PackageID): + package = self.__get_package(package_id=package_id) return deepcopy(package) - def save(self): - """ - Saves changes to initial `path`. - """ - print(f'⚙️ Saving {self.path}') - with open(self.path, 'w') as file: - json.dump( - self.packages, - fp=file, - indent=2, # preserve `swift package` indentation - sort_keys=True # preserve `swift package` packages sorting + def __get_package(self, package_id: PackageID): + pins = self.packages['object']['pins'] + package_pins = [index for index, p in enumerate(pins) if p['package'] == package_id.v1] + + if len(package_pins) == 0: + raise Exception( + f'{self.path} does not contain pin named "{package_id.v1}"' ) - file.write('\n') # add new line to the EOF - def print(self): - """ - Prints the content of this file. - """ - with open(self.path, 'r') as file: - print(f'⚙️ Content of {file.name}:') - print(file.read()) + package_pin_index = package_pins[0] + return self.packages['object']['pins'][package_pin_index] - def __get_package(self, package_name: str): - pins = self.packages['object']['pins'] - package_pins = [index for index, p in enumerate(pins) if p['package'] == package_name] + +class PackageResolvedContentV2(PackageResolvedContent): + """ + Example of `package.resolved` in version `2` looks this:: + + { + "pins" : [ + { + "identity" : "dd-sdk-ios", + "kind" : "remoteSourceControl", + "location" : "https://github.com/DataDog/dd-sdk-ios", + "state" : { + "branch" : "dogfooding", + "revision" : "6f662103771eb4523164e64f7f936bf9276f6bd0" + } + }, + ... + ] + "version" : 2 + } + + In v2 `branch` and `version` are mutually exclusive: if one is set, the other + is not present (unlike v1, where one was always set to `null`). + """ + + def __init__(self, path: str, json_content: dict): + self.path = path + self.packages = json_content + + def has_dependency(self, package_id: PackageID): + pins = self.packages['pins'] + return package_id.v2 in [p['identity'] for p in pins] + + def update_dependency(self, package_id: PackageID, new_branch: Optional[str], new_revision: str, new_version: Optional[str]): + package = self.__get_package(package_id=package_id) + + old_state = deepcopy(package['state']) + + if new_branch: + package['state']['branch'] = new_branch + else: + package['state'].pop('branch', None) # delete key regardless of whether it exists + + if new_revision: + package['state']['revision'] = new_revision + else: + package['state'].pop('revision', None) + + if new_version: + package['state']['version'] = new_version + else: + package['state'].pop('version', None) + + new_state = deepcopy(package['state']) + + diff = old_state.items() ^ new_state.items() + + if len(diff) > 0: + print(f'✏️️ Updated "{package_id.v2}" in {self.path}:') + print(f' → old: {old_state}') + print(f' → new: {new_state}') + else: + print(f'✏️️ "{package_id.v2}" is up-to-date in {self.path}') + + def add_dependency(self, package_id: PackageID, repository_url: str, branch: Optional[str], revision: str, version: Optional[str]): + pins = self.packages['pins'] + + # Find the index in `pins` array where the new dependency should be inserted. + # The `pins` array seems to follow the alphabetical order. + index = next((i for i in range(len(pins)) if pins[i]['identity'].lower() > package_id.v2.lower()), len(pins)) + + new_pin = { + 'identity': package_id.v2, + 'kind': 'remoteSourceControl', + 'location': repository_url, + 'state': {} + } + + if branch: + new_pin['state']['branch'] = branch + + if revision: + new_pin['state']['revision'] = revision + + if version: + new_pin['state']['version'] = version + + pins.insert(index, new_pin) + + print(f'✏️️ Added "{package_id.v2}" at index {index} in {self.path}:') + print(f' → branch: {branch}') + print(f' → revision: {revision}') + print(f' → version: {version}') + + def read_dependency_ids(self) -> [PackageID]: + pins = self.packages['pins'] + package_ids = [PackageID(v1=None, v2=pin['identity']) for pin in pins] + return package_ids + + def read_dependency(self, package_id: PackageID): + package = self.__get_package(package_id=package_id) + return deepcopy(package) + + def __get_package(self, package_id: PackageID): + pins = self.packages['pins'] + package_pins = [index for index, p in enumerate(pins) if p['identity'] == package_id.v2] if len(package_pins) == 0: raise Exception( - f'{self.path} does not contain pin named "{package_name}"' + f'{self.path} does not contain pin named "{package_id.v2}"' ) package_pin_index = package_pins[0] - return self.packages['object']['pins'][package_pin_index] + return self.packages['pins'][package_pin_index] diff --git a/tools/distribution/tests/dogfood/test_package_resolved.py b/tools/distribution/tests/dogfood/test_package_resolved.py new file mode 100644 index 0000000000..39835e51c6 --- /dev/null +++ b/tools/distribution/tests/dogfood/test_package_resolved.py @@ -0,0 +1,266 @@ +# ----------------------------------------------------------- +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-2020 Datadog, Inc. +# ----------------------------------------------------------- + + +import unittest +from tempfile import NamedTemporaryFile +from src.dogfood.package_resolved import PackageResolvedFile, PackageID, v2_package_id_from_repository_url + + +class PackageResolvedFileTestCase(unittest.TestCase): + v1_file_content = b''' + { + "object": { + "pins": [ + { + "package": "A", + "repositoryURL": "https://github.com/A-org/a.git", + "state": { + "branch": "a-branch", + "revision": "a-revision", + "version": null + } + }, + { + "package": "B", + "repositoryURL": "https://github.com/B-org/b.git", + "state": { + "branch": null, + "revision": "b-revision", + "version": "1.0.0" + } + } + ] + }, + "version": 1 + } + ''' + + v2_file_content = b''' + { + "pins" : [ + { + "identity" : "a", + "kind" : "remoteSourceControl", + "location" : "https://github.com/A-org/a", + "state" : { + "branch" : "a-branch", + "revision" : "a-revision" + } + }, + { + "identity" : "b", + "kind" : "remoteSourceControl", + "location" : "https://github.com/B-org/b.git", + "state" : { + "revision" : "b-revision", + "version" : "1.0.0" + } + } + ], + "version" : 2 + } + ''' + + def test_it_reads_version1_files(self): + with NamedTemporaryFile() as file: + file.write(self.v1_file_content) + file.seek(0) + + package_resolved = PackageResolvedFile(path=file.name) + self.assertTrue(package_resolved.has_dependency(package_id=PackageID(v1='A', v2='a'))) + self.assertTrue(package_resolved.has_dependency(package_id=PackageID(v1='B', v2='b'))) + self.assertFalse(package_resolved.has_dependency(package_id=PackageID(v1='C', v2='c'))) + self.assertListEqual( + [PackageID(v1='A', v2='a'), PackageID(v1='B', v2='b')], + package_resolved.read_dependency_ids() + ) + self.assertDictEqual( + { + 'package': 'A', + 'repositoryURL': 'https://github.com/A-org/a.git', + 'state': {'branch': 'a-branch', 'revision': 'a-revision', 'version': None} + }, + package_resolved.read_dependency(package_id=PackageID(v1='A', v2='a')) + ) + self.assertDictEqual( + { + 'package': 'B', + 'repositoryURL': 'https://github.com/B-org/b.git', + 'state': {'branch': None, 'revision': 'b-revision', 'version': '1.0.0'} + }, + package_resolved.read_dependency(package_id=PackageID(v1='B', v2='b')) + ) + + def test_it_changes_version1_files(self): + with NamedTemporaryFile() as file: + file.write(self.v1_file_content) + file.seek(0) + + package_resolved = PackageResolvedFile(path=file.name) + package_resolved.update_dependency( + package_id=PackageID(v1='B', v2='b'), + new_branch='b-branch-new', new_revision='b-revision-new', new_version=None + ) + package_resolved.add_dependency( + package_id=PackageID(v1='C', v2='c'), repository_url='https://github.com/C-org/c.git', + branch='c-branch', revision='c-revision', version=None + ) + package_resolved.add_dependency( + package_id=PackageID(v1='D', v2='d'), repository_url='https://github.com/D-org/d.git', + branch=None, revision='d-revision', version='1.1.0' + ) + package_resolved.save() + + actual_new_content = file.read().decode('utf-8') + expected_new_content = '''{ + "object": { + "pins": [ + { + "package": "A", + "repositoryURL": "https://github.com/A-org/a.git", + "state": { + "branch": "a-branch", + "revision": "a-revision", + "version": null + } + }, + { + "package": "B", + "repositoryURL": "https://github.com/B-org/b.git", + "state": { + "branch": "b-branch-new", + "revision": "b-revision-new", + "version": null + } + }, + { + "package": "C", + "repositoryURL": "https://github.com/C-org/c.git", + "state": { + "branch": "c-branch", + "revision": "c-revision", + "version": null + } + }, + { + "package": "D", + "repositoryURL": "https://github.com/D-org/d.git", + "state": { + "branch": null, + "revision": "d-revision", + "version": "1.1.0" + } + } + ] + }, + "version": 1 +} +''' + self.assertEqual(expected_new_content, actual_new_content) + + def test_it_reads_version2_files(self): + with NamedTemporaryFile() as file: + file.write(self.v2_file_content) + file.seek(0) + + package_resolved = PackageResolvedFile(path=file.name) + self.assertTrue(package_resolved.has_dependency(package_id=PackageID(v1=None, v2='a'))) + self.assertTrue(package_resolved.has_dependency(package_id=PackageID(v1=None, v2='b'))) + self.assertFalse(package_resolved.has_dependency(package_id=PackageID(v1=None, v2='c'))) + self.assertListEqual( + [PackageID(v1=None, v2='a'), PackageID(v1=None, v2='b')], + package_resolved.read_dependency_ids() + ) + self.assertDictEqual( + { + 'identity': 'a', + 'kind': 'remoteSourceControl', + 'location': 'https://github.com/A-org/a', + 'state': {'branch': 'a-branch', 'revision': 'a-revision'} + }, + package_resolved.read_dependency(package_id=PackageID(v1=None, v2='a')) + ) + self.assertDictEqual( + { + 'identity': 'b', + 'kind': 'remoteSourceControl', + 'location': 'https://github.com/B-org/b.git', + 'state': {'revision': 'b-revision', 'version': '1.0.0'} + }, + package_resolved.read_dependency(PackageID(v1=None, v2='b')) + ) + + def test_it_changes_version2_files(self): + with NamedTemporaryFile() as file: + file.write(self.v2_file_content) + file.seek(0) + + package_resolved = PackageResolvedFile(path=file.name) + package_resolved.update_dependency( + package_id=PackageID(v1=None, v2='b'), new_branch='b-branch-new', + new_revision='b-revision-new', new_version=None + ) + package_resolved.add_dependency( + package_id=PackageID(v1=None, v2='c'), repository_url='https://github.com/C-org/c.git', + branch='c-branch', revision='c-revision', version=None + ) + package_resolved.add_dependency( + package_id=PackageID(v1=None, v2='d'), repository_url='https://github.com/D-org/d.git', + branch=None, revision='d-revision', version='1.1.0' + ) + package_resolved.save() + + actual_new_content = file.read().decode('utf-8') + expected_new_content = '''{ + "pins" : [ + { + "identity" : "a", + "kind" : "remoteSourceControl", + "location" : "https://github.com/A-org/a", + "state" : { + "branch" : "a-branch", + "revision" : "a-revision" + } + }, + { + "identity" : "b", + "kind" : "remoteSourceControl", + "location" : "https://github.com/B-org/b.git", + "state" : { + "branch" : "b-branch-new", + "revision" : "b-revision-new" + } + }, + { + "identity" : "c", + "kind" : "remoteSourceControl", + "location" : "https://github.com/C-org/c.git", + "state" : { + "branch" : "c-branch", + "revision" : "c-revision" + } + }, + { + "identity" : "d", + "kind" : "remoteSourceControl", + "location" : "https://github.com/D-org/d.git", + "state" : { + "revision" : "d-revision", + "version" : "1.1.0" + } + } + ], + "version" : 2 +} +''' + self.assertEqual(expected_new_content, actual_new_content) + + def test_v2_package_id_from_repository_url(self): + self.assertEqual('abc', v2_package_id_from_repository_url(repository_url='https://github.com/A-org/abc.git')) + self.assertEqual('abc', v2_package_id_from_repository_url(repository_url='https://github.com/A-org/abc')) + self.assertEqual('abc', v2_package_id_from_repository_url(repository_url='git@github.com:DataDog/abc.git')) + self.assertEqual('abc', v2_package_id_from_repository_url(repository_url='git@github.com:DataDog/abc'))