Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci: feat: check for unused files and restrict extensions #2678

Merged
merged 48 commits into from
Dec 1, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
9eb90df
hpp
AJPfleger Nov 15, 2023
89401dd
cpp ipp
AJPfleger Nov 15, 2023
b0de783
images
AJPfleger Nov 15, 2023
b2a8681
revert
AJPfleger Nov 15, 2023
4a99a97
Revert "hpp"
AJPfleger Nov 16, 2023
af9abf0
Revert "cpp ipp"
AJPfleger Nov 16, 2023
f9aef99
Merge branch 'main' into unused
AJPfleger Nov 16, 2023
dc0c33d
Core
AJPfleger Nov 16, 2023
a6d0034
Examples/Io
AJPfleger Nov 16, 2023
473f07a
DetectorInspectorsStub
AJPfleger Nov 16, 2023
c2809be
Tests/UnitTests/Core/Seeding
AJPfleger Nov 16, 2023
511687d
exatrkx
AJPfleger Nov 16, 2023
e534b87
other tests
AJPfleger Nov 16, 2023
1a12ce6
Plugins/DD4hep/src/DD4hepMaterialConverter.cpp
AJPfleger Nov 16, 2023
40ff8a8
Restore "exatrkx test"
AJPfleger Nov 16, 2023
35a3387
exatrkx
AJPfleger Nov 16, 2023
e87695c
add CI-job
AJPfleger Nov 17, 2023
8690a57
Revert "add CI-job"
AJPfleger Nov 17, 2023
340b26d
add python-script
AJPfleger Nov 17, 2023
65e377c
Merge branch 'main' into unused
AJPfleger Nov 17, 2023
a1368d9
add CI-job
AJPfleger Nov 17, 2023
abca40f
update script
AJPfleger Nov 17, 2023
71adb1b
comment rm
AJPfleger Nov 17, 2023
4eea04a
Empty-Commit
AJPfleger Nov 17, 2023
0cdb79a
change permission
AJPfleger Nov 17, 2023
1da5d3e
change permission again
AJPfleger Nov 17, 2023
c20e825
remove '.h'
AJPfleger Nov 17, 2023
440187a
unify jpeg->jpg
AJPfleger Nov 17, 2023
9eac14f
unify .h->.hpp
AJPfleger Nov 17, 2023
57e1129
fix
AJPfleger Nov 17, 2023
5177c2b
remove fail-test
AJPfleger Nov 17, 2023
1769d11
simplify logic
AJPfleger Nov 17, 2023
2b105c2
Merge branch 'acts-project:main' into unused
AJPfleger Nov 17, 2023
6655247
Revert "remove fail-test"
AJPfleger Nov 17, 2023
b30fee2
Revert "unify .h->.hpp"
AJPfleger Nov 17, 2023
930b902
exclude sycl .h-file
AJPfleger Nov 18, 2023
9b69825
Revert "polyhedron"
AJPfleger Nov 20, 2023
794a5ea
rm cpp ipp
AJPfleger Nov 20, 2023
a665b25
Merge branch 'main' into unused
AJPfleger Nov 20, 2023
ed22119
RGB <3
AJPfleger Nov 20, 2023
759513f
size_t
AJPfleger Nov 20, 2023
5b9e5f0
missing spaces
AJPfleger Nov 20, 2023
95eb047
progress in output
AJPfleger Nov 21, 2023
1e8149c
Merge branch 'main' into unused
AJPfleger Nov 21, 2023
07b387c
Revert "rm cpp ipp"
AJPfleger Dec 1, 2023
17b92ca
Merge branch 'main' into unused
AJPfleger Dec 1, 2023
81a0792
remove cpp ipp again
AJPfleger Dec 1, 2023
6a539e1
Merge branch 'main' into unused
kodiakhq[bot] Dec 1, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .github/workflows/checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -150,3 +150,13 @@ jobs:
- name: Check
run: >
CI/check_fpe_masks.py --token ${{ secrets.GITHUB_TOKEN }}
unused_files:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Check
run: >
CI/check_unused_files.py
228 changes: 228 additions & 0 deletions CI/check_unused_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
#!/usr/bin/env python3

from pathlib import Path
import os
import sys
import subprocess


def file_can_be_removed(searchstring, scope):
cmd = "grep -IR '" + searchstring + "' " + " ".join(scope)

p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
output, _ = p.communicate()
return output == b""


def count_files(path=".", exclude_dirs=(), exclude_files=()):
count = 0
for root, dirs, files in os.walk(path):
dirs[:] = [d for d in dirs if d not in exclude_dirs]
files[:] = [f for f in files if f not in exclude_files]
count += len(files)

return count


def main():
print("\033[32mINFO\033[0m Start check_unused_files.py ...")
exclude_dirs = (
"Scripts",
"thirdparty",
"CI",
"git",
"cmake",
".git",
".github",
".",
".idea",
)
exclude_files = (
"acts_logo_colored.svg",
".gitignore",
"README.md",
"CMakeLists.txt",
"DetUtils.h",
"CommandLineArguments.h",
# Filename not completed in source
"vertexing_event_mu20_beamspot.csv",
"vertexing_event_mu20_tracks.csv",
"vertexing_event_mu20_vertices_AMVF.csv",
# TODO Move the following files to a better place?
"Magfield.ipynb",
"SolenoidField.ipynb",
# TODO Add README next to the following files?
"default-input-config-generic.json",
"geoSelection-openDataDetector.json",
"alignment-geo-contextualDetector.json",
)

suffix_header = (
".hpp",
".cuh",
)
suffix_source = (
".ipp",
".cpp",
".cu",
)
suffix_image = (
".png",
".svg",
".jpg",
".gif",
)
suffix_python = (".py",)
suffix_doc = (
".md",
".rst",
)
suffix_other = (
"",
".csv",
".css",
".gdml",
".hepmc3",
".in",
".ipynb",
".json",
".j2",
".onnx",
".root",
".toml",
".txt",
".yml",
)
suffix_allowed = (
suffix_header
+ suffix_source
+ suffix_image
+ suffix_python
+ suffix_doc
+ suffix_other
)

exit = 0

dirs_base = next(os.walk("."))[1]
dirs_base[:] = [d for d in dirs_base if d not in exclude_dirs]
dirs_base_docs = ("docs",)
dirs_base_code = [d for d in dirs_base if d not in dirs_base_docs]

# Collectors
wrong_extension = ()
unused_files = ()

# walk over all files
for root, dirs, files in os.walk("."):
dirs[:] = [d for d in dirs if d not in exclude_dirs]
files[:] = [f for f in files if f not in exclude_files]

# Skip base-directory
if str(Path(root)) == ".":
continue

# Print progress
if root[2:] in dirs_base:
processed_files = 0
current_base_dir = root
number_files = count_files(root, exclude_dirs, exclude_files)
# print empty to start a new line
print("")

# Skip "white-paper-figures"
# TODO Find a more elegant way
if str(root).find("white_papers/figures") != -1:
processed_files += count_files(root, exclude_dirs, exclude_files)
continue

# Skip "DD4hep-tests" since their cmake looks a bit different
# TODO Find a more elegant way
if str(root).find("Tests/UnitTests/Plugins/DD4hep") != -1:
processed_files += count_files(root, exclude_dirs, exclude_files)
continue

root = Path(root)
for filename in files:
processed_files += 1
# get the full path of the file
filepath = root / filename

# Check for wrong extensions
if filepath.suffix not in suffix_allowed:
wrong_extension += (str(filepath),)

# Check header files and remove
if filepath.suffix in suffix_header + suffix_source:
if file_can_be_removed(filepath.stem, dirs_base_code):
unused_files += (str(filepath),)
remove_cmd = "rm " + str(filepath)
os.system(remove_cmd)

# TODO Find test to check python files
if filepath.suffix in suffix_python:
continue

# Check documentation files (weak tests)
# TODO find more reliable test for this
if filepath.suffix in suffix_doc:
if file_can_be_removed(filepath.stem, dirs_base_docs):
unused_files += (str(filepath),)
remove_cmd = "rm " + str(filepath)
os.system(remove_cmd)

# Check and print other files
if filepath.suffix in suffix_image + suffix_other:
if file_can_be_removed(filename, dirs_base):
unused_files += (str(filepath),)
remove_cmd = "rm " + str(filepath)
os.system(remove_cmd)

# Print the progress in place
progress = int(20 * processed_files / number_files)
sys.stdout.write("\r")
sys.stdout.write(
"Checked [%-20s] %d/%d files in %s"
% ("=" * progress, processed_files, number_files, current_base_dir)
)
sys.stdout.flush()

if len(wrong_extension) != 0:
print(
"\n\n\033[31mERROR\033[0m "
+ f"The following {len(wrong_extension)} files have an unsupported extension:\n\n"
+ "\033[31m"
+ "\n".join(wrong_extension)
+ "\033[0m"
+ "\nCheck if you can change the format to one of the following:\n"
+ "\n".join(suffix_allowed)
+ "\nIf you really need that specific extension, add it to the list above.\n"
)

exit += 1

if len(unused_files) != 0:
print(
"\n\n\033[31mERROR\033[0m "
+ f"The following {len(unused_files)} files seem to be unused:\n"
+ "\033[31m"
+ "\n".join(unused_files)
+ "\033[0m"
+ "\nYou have 3 options:"
+ "\n\t- Remove them"
+ "\n\t- Use them (check proper include)"
+ "\n\t- Modify the ignore list of this check\n"
)

exit += 1

if exit == 0:
print(
"\n\n\033[32mINFO\033[0m Finished check_unused_files.py without any errors."
)

return exit


if "__main__" == __name__:
sys.exit(main())
145 changes: 0 additions & 145 deletions Core/include/Acts/Material/MaterialCollector.hpp

This file was deleted.

Loading
Loading