Skip to content

Commit

Permalink
Refactor (formatter): apply yapf changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@Giuseppe5
Giuseppe5 committed Mar 28, 2023
1 parent e87f318 commit e5cb7b5
Show file tree
Hide file tree
Showing 264 changed files with 5,284 additions and 3,668 deletions.
198 changes: 110 additions & 88 deletions .github/workflows/gen_github_actions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,22 @@

PLATFORM_LIST_REDUCED = ['ubuntu-latest']

MATRIX_REDUCED = od([('python_version', list(PYTHON_VERSIONS_REDUCED)), ('pytorch_version', list(PYTORCH_LIST_REDUCED)),
('platform', PLATFORM_LIST_REDUCED)])
MATRIX_REDUCED = od([('python_version', list(PYTHON_VERSIONS_REDUCED)),
('pytorch_version', list(PYTORCH_LIST_REDUCED)),
('platform', PLATFORM_LIST_REDUCED)])

FINN_MATRIX_REDUCED = od([('python_version', list(PYTHON_VERSIONS_REDUCED)),
('pytorch_version', list(PYTORCH_LIST_REDUCED)), ('platform', PLATFORM_LIST_REDUCED)])
('pytorch_version', list(PYTORCH_LIST_REDUCED)),
('platform', PLATFORM_LIST_REDUCED)])

PYTEST_MATRIX_EXTRA_REDUCED = od([('jit_status', ['jit_disabled',])])
PYTEST_MATRIX_EXTRA_REDUCED = od([('jit_status', [
'jit_disabled',])])

# Data shared betwen Nox sessions and Github Actions, formatted as tuples
PYTHON_VERSIONS = ('3.7', '3.8')

PYTORCH_VERSIONS = ('1.5.1', '1.6.0', '1.7.1', '1.8.1', '1.9.1', '1.10.1', '1.11.0', '1.12.1',
'1.13.0')
PYTORCH_VERSIONS = (
'1.5.1', '1.6.0', '1.7.1', '1.8.1', '1.9.1', '1.10.1', '1.11.0', '1.12.1', '1.13.0')
JIT_STATUSES = ('jit_disabled', 'jit_enabled')

# Data used only by Github Actions, formatted as lists or lists of ordered dicts
Expand All @@ -44,94 +47,92 @@

EXCLUDE_LIST = []

JIT_EXCLUDE_LIST = generate_exclusion_list([[
['pytorch_version',['1.5.1', '1.6.0', '1.7.1', '1.8.1', '1.9.1']],
['jit_status', ['jit_enabled', ]]]])
JIT_EXCLUDE_LIST = generate_exclusion_list([[[
'pytorch_version', ['1.5.1', '1.6.0', '1.7.1', '1.8.1', '1.9.1']],
['jit_status', [
'jit_enabled',]]]])

NOTEBOOK_EXCLUDE_LIST = generate_exclusion_list( [
[['pytorch_version',['1.5.1', '1.6.0', '1.7.1']]],
[['platform', ['macos-latest',]]]])
NOTEBOOK_EXCLUDE_LIST = generate_exclusion_list([[['pytorch_version', ['1.5.1', '1.6.0', '1.7.1']]],
[['platform', [
'macos-latest',]]]])

END_TO_END_EXCLUDE_LIST = [od([('platform', ['windows-latest',])])]
END_TO_END_EXCLUDE_LIST = [od([('platform', [
'windows-latest',])])]

MATRIX = od([('python_version', list(PYTHON_VERSIONS)),
('pytorch_version', list(PYTORCH_VERSIONS)),
MATRIX = od([('python_version', list(PYTHON_VERSIONS)), ('pytorch_version', list(PYTORCH_VERSIONS)),
('platform', PLATFORM_LIST)])

FINN_MATRIX = od([('python_version', list(PYTHON_VERSIONS)),
('pytorch_version', list(PYTORCH_VERSIONS)),
('platform', FINN_PLATFORM_LIST)])
('pytorch_version', list(PYTORCH_VERSIONS)), ('platform', FINN_PLATFORM_LIST)])

PYTEST_MATRIX_EXTRA = od([('jit_status', list(JIT_STATUSES))])

PYTEST_STEP_LIST = [
od([('name', 'Run Nox session for brevitas pytest'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_cpu-${{ matrix.python_version }}\(${{ matrix.jit_status }}\,\ pytorch_${{ matrix.pytorch_version }}\)')]),
]
od([('name', 'Run Nox session for brevitas pytest'), ('shell', 'bash'),
(
'run',
'nox -v -s tests_brevitas_cpu-${{ matrix.python_version }}\(${{ matrix.jit_status }}\,\ pytorch_${{ matrix.pytorch_version }}\)'
)]),]

EXAMPLES_PYTEST_STEP_LIST = [
od([('name', 'Run Nox session for brevitas_examples pytest'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_examples_cpu-${{ matrix.python_version }}\(${{ matrix.jit_status }}\,\ pytorch_${{ matrix.pytorch_version }}\)')]),
]
od([('name', 'Run Nox session for brevitas_examples pytest'), ('shell', 'bash'),
(
'run',
'nox -v -s tests_brevitas_examples_cpu-${{ matrix.python_version }}\(${{ matrix.jit_status }}\,\ pytorch_${{ matrix.pytorch_version }}\)'
)]),]

FINN_INTEGRATION_STEP_LIST = [
od([
('name', 'Install protobuf on Ubuntu'),
('shell', 'bash'),
('run',
'sudo apt-get install protobuf-compiler libprotoc-dev'),
('if', "startsWith(runner.os, 'Linux') == true")
]),
od([
('name', 'Run Nox session for Brevitas-FINN integration'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_finn_integration-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)')
])]
od([('name', 'Install protobuf on Ubuntu'), ('shell', 'bash'),
('run', 'sudo apt-get install protobuf-compiler libprotoc-dev'),
('if', "startsWith(runner.os, 'Linux') == true")]),
od([('name', 'Run Nox session for Brevitas-FINN integration'), ('shell', 'bash'),
(
'run',
'nox -v -s tests_brevitas_finn_integration-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)'
)])]

PYXIR_INTEGRATION_STEP_LIST = [
od([('name', 'Run Nox session for Brevitas-PyXIR integration'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_pyxir_integration-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)')
])]
od([('name', 'Run Nox session for Brevitas-PyXIR integration'), ('shell', 'bash'),
(
'run',
'nox -v -s tests_brevitas_pyxir_integration-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)'
)])]

ORT_INTEGRATION_STEP_LIST = [
od([('name', 'Run Nox session for Brevitas-ORT integration'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_ort_integration-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)')
])]
od([('name', 'Run Nox session for Brevitas-ORT integration'), ('shell', 'bash'),
(
'run',
'nox -v -s tests_brevitas_ort_integration-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)'
)])]

TEST_INSTALL_DEV_STEP_LIST = [
od([('name', 'Run Nox session for testing brevitas develop install and imports'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_install_dev-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)'
(
'run',
'nox -v -s tests_brevitas_install_dev-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)'
)]),
od([('name', 'Run Nox session for testing brevitas_examples develop install and imports'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_examples_install_dev-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)')
])]
(
'run',
'nox -v -s tests_brevitas_examples_install_dev-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)'
)])]

NOTEBOOK_STEP_LIST = [
od([('name', 'Run Nox session for Notebook execution'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_notebook-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)')
])]
od([('name', 'Run Nox session for Notebook execution'), ('shell', 'bash'),
(
'run',
'nox -v -s tests_brevitas_notebook-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)'
)])]

ENDTOEND_STEP_LIST = [
od([('name', 'Run Nox session for end-to-end flows'),
('shell', 'bash'),
('run',
'nox -v -s tests_brevitas_end_to_end-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)')
])]
od([('name', 'Run Nox session for end-to-end flows'), ('shell', 'bash'),
(
'run',
'nox -v -s tests_brevitas_end_to_end-${{ matrix.python_version }}\(\pytorch_${{ matrix.pytorch_version }}\)'
)])]


def gen_pytest_yml():
pytest = Action(
Expand All @@ -141,10 +142,15 @@ def gen_pytest_yml():
PYTEST_STEP_LIST,
STRATEGY)
pytest.gen_yaml(BASE_YML_TEMPLATE, PYTEST_YML)
pytest = Action('Pytest', EXCLUDE_LIST, combine_od_list([MATRIX_REDUCED, PYTEST_MATRIX_EXTRA_REDUCED]),
PYTEST_STEP_LIST, STRATEGY)
pytest = Action(
'Pytest',
EXCLUDE_LIST,
combine_od_list([MATRIX_REDUCED, PYTEST_MATRIX_EXTRA_REDUCED]),
PYTEST_STEP_LIST,
STRATEGY)
pytest.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + PYTEST_YML)


def gen_examples_pytest_yml():
pytest = Action(
'Examples Pytest',
Expand All @@ -153,22 +159,24 @@ def gen_examples_pytest_yml():
EXAMPLES_PYTEST_STEP_LIST,
STRATEGY)
pytest.gen_yaml(BASE_YML_TEMPLATE, EXAMPLES_PYTEST_YML)
pytest = Action('Examples Pytest', EXCLUDE_LIST, combine_od_list([MATRIX_REDUCED, PYTEST_MATRIX_EXTRA_REDUCED]),
EXAMPLES_PYTEST_STEP_LIST, STRATEGY)
pytest = Action(
'Examples Pytest',
EXCLUDE_LIST,
combine_od_list([MATRIX_REDUCED, PYTEST_MATRIX_EXTRA_REDUCED]),
EXAMPLES_PYTEST_STEP_LIST,
STRATEGY)
pytest.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + EXAMPLES_PYTEST_YML)


def gen_test_develop_install_yml():
test_develop_install = Action(
'Test develop install',
EXCLUDE_LIST,
MATRIX,
TEST_INSTALL_DEV_STEP_LIST,
STRATEGY)
'Test develop install', EXCLUDE_LIST, MATRIX, TEST_INSTALL_DEV_STEP_LIST, STRATEGY)
test_develop_install.gen_yaml(BASE_YML_TEMPLATE, DEVELOP_INSTALL_YML)
test_develop_install = Action('Test develop install', EXCLUDE_LIST, MATRIX_REDUCED,
TEST_INSTALL_DEV_STEP_LIST, STRATEGY)
test_develop_install = Action(
'Test develop install', EXCLUDE_LIST, MATRIX_REDUCED, TEST_INSTALL_DEV_STEP_LIST, STRATEGY)
test_develop_install.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + DEVELOP_INSTALL_YML)


def gen_test_brevitas_finn_integration():
test_finn_integration = Action(
'Test Brevitas-FINN integration',
Expand All @@ -177,22 +185,28 @@ def gen_test_brevitas_finn_integration():
FINN_INTEGRATION_STEP_LIST,
STRATEGY)
test_finn_integration.gen_yaml(BASE_YML_TEMPLATE, FINN_INTEGRATION_YML)
test_finn_integration = Action('Test Brevitas-FINN integration', EXCLUDE_LIST, FINN_MATRIX_REDUCED,
FINN_INTEGRATION_STEP_LIST, STRATEGY)
test_finn_integration = Action(
'Test Brevitas-FINN integration',
EXCLUDE_LIST,
FINN_MATRIX_REDUCED,
FINN_INTEGRATION_STEP_LIST,
STRATEGY)
test_finn_integration.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + FINN_INTEGRATION_YML)


def gen_test_brevitas_ort_integration():
test_ort_integration = Action(
'Test Brevitas-ORT integration', EXCLUDE_LIST, MATRIX, ORT_INTEGRATION_STEP_LIST, STRATEGY)
test_ort_integration.gen_yaml(BASE_YML_TEMPLATE, ORT_INTEGRATION_YML)
test_ort_integration = Action(
'Test Brevitas-ORT integration',
EXCLUDE_LIST,
MATRIX,
MATRIX_REDUCED,
ORT_INTEGRATION_STEP_LIST,
STRATEGY)
test_ort_integration.gen_yaml(BASE_YML_TEMPLATE, ORT_INTEGRATION_YML)
test_ort_integration = Action('Test Brevitas-ORT integration', EXCLUDE_LIST, MATRIX_REDUCED,
ORT_INTEGRATION_STEP_LIST, STRATEGY)
test_ort_integration.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + ORT_INTEGRATION_YML)


def gen_test_brevitas_notebook():
tests_brevitas_notebooks = Action(
'Test Notebook execution',
Expand All @@ -201,11 +215,15 @@ def gen_test_brevitas_notebook():
NOTEBOOK_STEP_LIST,
STRATEGY)
tests_brevitas_notebooks.gen_yaml(BASE_YML_TEMPLATE, NOTEBOOK_YML)
tests_brevitas_notebooks = Action('Test Notebook execution',
EXCLUDE_LIST + NOTEBOOK_EXCLUDE_LIST, MATRIX_REDUCED,
NOTEBOOK_STEP_LIST, STRATEGY)
tests_brevitas_notebooks = Action(
'Test Notebook execution',
EXCLUDE_LIST + NOTEBOOK_EXCLUDE_LIST,
MATRIX_REDUCED,
NOTEBOOK_STEP_LIST,
STRATEGY)
tests_brevitas_notebooks.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + NOTEBOOK_YML)


def gen_test_brevitas_end_to_end():
tests_brevitas_end_to_end = Action(
'Test End-to-end flows',
Expand All @@ -214,11 +232,15 @@ def gen_test_brevitas_end_to_end():
ENDTOEND_STEP_LIST,
STRATEGY)
tests_brevitas_end_to_end.gen_yaml(BASE_YML_TEMPLATE, ENDTOEND_YML)
tests_brevitas_end_to_end = Action('Test End-to-end flows',
EXCLUDE_LIST + END_TO_END_EXCLUDE_LIST, MATRIX_REDUCED,
ENDTOEND_STEP_LIST, STRATEGY)
tests_brevitas_end_to_end = Action(
'Test End-to-end flows',
EXCLUDE_LIST + END_TO_END_EXCLUDE_LIST,
MATRIX_REDUCED,
ENDTOEND_STEP_LIST,
STRATEGY)
tests_brevitas_end_to_end.gen_yaml(BASE_YML_REDUCED_TEMPLATE, 'reduced_' + ENDTOEND_YML)


if __name__ == '__main__':
gen_pytest_yml()
gen_examples_pytest_yml()
Expand Down
9 changes: 5 additions & 4 deletions .github/workflows/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def dict_str(d, quote_val, indent_first, newline_val=False):
elif newline_val:
for i, v in enumerate(val):
repr += f"{name}: '{v}'" + NIX_NEWLINE
if i < len(val)-1:
if i < len(val) - 1:
repr += first_line_prefix
else:
repr += f"{name}: {val}" + NIX_NEWLINE
Expand All @@ -58,13 +58,13 @@ def dict_str(d, quote_val, indent_first, newline_val=False):
def gen_yaml(self, base_template_path, output_path):
d = {}
d['name'] = self.name
d['matrix'] = indent(
Action.dict_str(self.matrix, False, False), MATRIX_INDENT * ' ')
d['matrix'] = indent(Action.dict_str(self.matrix, False, False), MATRIX_INDENT * ' ')
d['steps'] = indent(
Action.list_of_dicts_str(self.step_list, False, True), STEP_INDENT * ' ')
if self.exclude_list:
d['exclude'] = indent('exclude:\n', MATRIX_INDENT * ' ') + indent(
Action.list_of_dicts_str(self.exclude_list, False, True, True), EXCLUDE_INDENT * ' ')
Action.list_of_dicts_str(self.exclude_list, False, True, True),
EXCLUDE_INDENT * ' ')
else:
d['exclude'] = ''

Expand All @@ -79,6 +79,7 @@ def gen_yaml(self, base_template_path, output_path):
def combine_od_list(od_list):
return od(reduce(lambda l1, l2: l1 + l2, list(map(lambda d: list(d.items()), od_list))))


def generate_exclusion_list(combinations):
"""
Takes as input a List(first level) of List(second level) of List[str, List] (options)
Expand Down
Loading

0 comments on commit e5cb7b5

Please sign in to comment.