Skip to content

Commit

Permalink
Update CI to newer changes (#167)
Browse files Browse the repository at this point in the history
  • Loading branch information
jborean93 authored Nov 26, 2024
1 parent c950082 commit 8f2820c
Show file tree
Hide file tree
Showing 14 changed files with 114 additions and 58 deletions.
11 changes: 3 additions & 8 deletions .azure-pipelines/azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,12 @@ variables:
value: ansible_collections/microsoft/ad
- name: coverageBranches
value: main
- name: pipelinesCoverage
value: coverage-powershell
- name: entryPoint
value: tests/utils/shippable/shippable.sh
value: .azure-pipelines/commands/entry-point.sh
- name: fetchDepth
value: 0

resources:
containers:
- container: default
image: quay.io/ansible/azure-pipelines-test-container:6.0.0
- name: defaultContainer
value: quay.io/ansible/azure-pipelines-test-container:6.0.0

pool: Standard

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,4 +97,4 @@ fi

ansible-test env --dump --show --timeout "${timeout}" --color -v

"tests/utils/shippable/${script}.sh" "${test}"
".azure-pipelines/commands/${script}.sh" "${test}"
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
3 changes: 1 addition & 2 deletions .azure-pipelines/scripts/combine-coverage.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
It is up to pipeline authors to avoid name collisions when deviating from the recommended format.
"""

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from __future__ import annotations

import os
import re
Expand Down
102 changes: 102 additions & 0 deletions .azure-pipelines/scripts/publish-codecov.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
#!/usr/bin/env python
"""
Upload code coverage reports to codecov.io.
Multiple coverage files from multiple languages are accepted and aggregated after upload.
Python coverage, as well as PowerShell and Python stubs can all be uploaded.
"""
from __future__ import annotations

import argparse
import dataclasses
import pathlib
import shutil
import subprocess
import tempfile
import typing as t
import urllib.request


@dataclasses.dataclass(frozen=True)
class CoverageFile:
name: str
path: pathlib.Path
flags: t.List[str]


@dataclasses.dataclass(frozen=True)
class Args:
dry_run: bool
path: pathlib.Path


def parse_args() -> Args:
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--dry-run', action='store_true')
parser.add_argument('path', type=pathlib.Path)

args = parser.parse_args()

# Store arguments in a typed dataclass
fields = dataclasses.fields(Args)
kwargs = {field.name: getattr(args, field.name) for field in fields}

return Args(**kwargs)


def process_files(directory: pathlib.Path) -> t.Tuple[CoverageFile, ...]:
processed = []
for file in directory.joinpath('reports').glob('coverage*.xml'):
name = file.stem.replace('coverage=', '')

# Get flags from name
flags = name.replace('-powershell', '').split('=') # Drop '-powershell' suffix
flags = [flag if not flag.startswith('stub') else flag.split('-')[0] for flag in flags] # Remove "-01" from stub files

processed.append(CoverageFile(name, file, flags))

return tuple(processed)


def upload_files(codecov_bin: pathlib.Path, files: t.Tuple[CoverageFile, ...], dry_run: bool = False) -> None:
for file in files:
cmd = [
str(codecov_bin),
'--name', file.name,
'--file', str(file.path),
]
for flag in file.flags:
cmd.extend(['--flags', flag])

if dry_run:
print(f'DRY-RUN: Would run command: {cmd}')
continue

subprocess.run(cmd, check=True)


def download_file(url: str, dest: pathlib.Path, flags: int, dry_run: bool = False) -> None:
if dry_run:
print(f'DRY-RUN: Would download {url} to {dest} and set mode to {flags:o}')
return

with urllib.request.urlopen(url) as resp:
with dest.open('w+b') as f:
# Read data in chunks rather than all at once
shutil.copyfileobj(resp, f, 64 * 1024)

dest.chmod(flags)


def main():
args = parse_args()
url = 'https://ci-files.testing.ansible.com/codecov/linux/codecov'
with tempfile.TemporaryDirectory(prefix='codecov-') as tmpdir:
codecov_bin = pathlib.Path(tmpdir) / 'codecov'
download_file(url, codecov_bin, 0o755, args.dry_run)

files = process_files(args.path)
upload_files(codecov_bin, files, args.dry_run)


if __name__ == '__main__':
main()
27 changes: 0 additions & 27 deletions .azure-pipelines/scripts/publish-codecov.sh

This file was deleted.

2 changes: 1 addition & 1 deletion .azure-pipelines/scripts/report-coverage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ fi

# Generate stubs using docker (if supported) otherwise fall back to using a virtual environment instead.
# The use of docker is required when Powershell code is present, but Ansible 2.12 was the first version to support --docker with coverage.
ansible-test coverage xml --stub --docker --color -v || ansible-test coverage xml --stub --venv --venv-system-site-packages --color -v
ansible-test coverage xml --group-by command --stub --docker --color -v || ansible-test coverage xml --group-by command --stub --venv --color -v
3 changes: 1 addition & 2 deletions .azure-pipelines/scripts/time-command.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#!/usr/bin/env python
"""Prepends a relative timestamp to each input line from stdin and writes it to stdout."""

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from __future__ import annotations

import sys
import time
Expand Down
17 changes: 2 additions & 15 deletions .azure-pipelines/templates/coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
jobs:
- job: Coverage
displayName: Code Coverage
container: default
container: $[ variables.defaultContainer ]
workspace:
clean: all
steps:
Expand All @@ -23,20 +23,7 @@ jobs:
- bash: .azure-pipelines/scripts/report-coverage.sh
displayName: Generate Coverage Report
condition: gt(variables.coverageFileCount, 0)
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
# Azure Pipelines only accepts a single coverage data file.
# That means only Python or PowerShell coverage can be uploaded, but not both.
# Set the "pipelinesCoverage" variable to determine which type is uploaded.
# Use "coverage" for Python and "coverage-powershell" for PowerShell.
summaryFileLocation: "$(outputPath)/reports/$(pipelinesCoverage).xml"
# Override the root (sources) path specified in the coverage XML files.
# This allows coverage to be reported in Azure Pipelines even if the report was generated in a container.
pathToSources: "$(Agent.BuildDirectory)/$(checkoutPath)"
displayName: Publish to Azure Pipelines
condition: gt(variables.coverageFileCount, 0)
- bash: .azure-pipelines/scripts/publish-codecov.sh "$(outputPath)"
- bash: .azure-pipelines/scripts/publish-codecov.py "$(outputPath)"
displayName: Publish to codecov.io
condition: gt(variables.coverageFileCount, 0)
continueOnError: true
2 changes: 1 addition & 1 deletion .azure-pipelines/templates/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
- ${{ each job in parameters.jobs }}:
- job: test_${{ replace(replace(replace(replace(job.test, '/', '_'), '.', '_'), '-', '_'), '@', '_') }}
displayName: ${{ job.name }}
container: default
container: $[ variables.defaultContainer ]
workspace:
clean: all
steps:
Expand Down
3 changes: 2 additions & 1 deletion tests/sanity/ignore-2.15.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
plugins/action/domain.py action-plugin-docs # ansible-test is ignoring sidecar docs
plugins/action/domain_child.py action-plugin-docs # ansible-test is ignoring sidecar docs
plugins/action/domain_controller.py action-plugin-docs # ansible-test is ignoring sidecar docs
plugins/action/membership.py action-plugin-docs # ansible-test is ignoring sidecar docs
plugins/action/membership.py action-plugin-docs # ansible-test is ignoring sidecar docs
.azure-pipelines/scripts/publish-codecov.py replace-urlopen # fixed in newer Ansible versions

0 comments on commit 8f2820c

Please sign in to comment.