From c1667adf3d2220d3f1e19742586984a182721a26 Mon Sep 17 00:00:00 2001 From: joey0320 Date: Thu, 27 Apr 2023 20:39:07 -0700 Subject: [PATCH 01/10] trial 1 --- common.mk | 20 ++-- scripts/split-module-files.py | 111 --------------------- scripts/uniqify-module-names.py | 170 +++++++++++++++++++++++++++----- 3 files changed, 153 insertions(+), 148 deletions(-) delete mode 100755 scripts/split-module-files.py diff --git a/common.mk b/common.mk index 5c1f2461d1..4aeb44aa2d 100644 --- a/common.mk +++ b/common.mk @@ -231,26 +231,22 @@ $(SFC_MFC_TARGETS) &: $(FIRRTL_FILE) $(FINAL_ANNO_FILE) $(SFC_LEVEL) $(EXTRA_FIR $(SED) -i 's/.*/& /' $(MFC_SMEMS_CONF) # need trailing space for SFC macrocompiler # DOC include end: FirrtlCompiler -$(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(ALL_MODS_FILELIST) $(BB_MODS_FILELIST) $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) &: $(MFC_MODEL_HRCHY_JSON) $(MFC_FILELIST) $(MFC_BB_MODS_FILELIST) - $(base_dir)/scripts/split-module-files.py \ +$(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(ALL_MODS_FILELIST) $(BB_MODS_FILELIST) $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) &: $(MFC_MODEL_HRCHY_JSON) $(MFC_TOP_HRCHY_JSON) $(MFC_FILELIST) $(MFC_BB_MODS_FILELIST) + $(base_dir)/scripts/uniqify-module-names.py \ --model-hier-json $(MFC_MODEL_HRCHY_JSON) \ + --top-hier-json $(MFC_TOP_HRCHY_JSON) \ + --in-all-filelist $(MFC_FILELIST) \ --dut $(TOP) \ + --model $(MODEL) \ + --target-dir $(GEN_COLLATERAL_DIR) \ --out-dut-filelist $(TOP_MODS_FILELIST) \ --out-model-filelist $(MODEL_MODS_FILELIST) \ - --in-all-filelist $(MFC_FILELIST) \ - --target-dir $(GEN_COLLATERAL_DIR) + --out-model-hier-json $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) \ + --gcpath $(GEN_COLLATERAL_DIR) $(SED) -e 's;^;$(GEN_COLLATERAL_DIR)/;' $(MFC_BB_MODS_FILELIST) > $(BB_MODS_FILELIST) $(SED) -i 's/\.\///' $(TOP_MODS_FILELIST) $(SED) -i 's/\.\///' $(MODEL_MODS_FILELIST) $(SED) -i 's/\.\///' $(BB_MODS_FILELIST) - $(base_dir)/scripts/uniqify-module-names.py \ - --top-filelist $(TOP_MODS_FILELIST) \ - --mod-filelist $(MODEL_MODS_FILELIST) \ - --gen-collateral-path $(GEN_COLLATERAL_DIR) \ - --model-hier-json $(MFC_MODEL_HRCHY_JSON) \ - --out-model-hier-json $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) \ - --dut $(TOP) \ - --model $(MODEL) sort -u $(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(BB_MODS_FILELIST) > $(ALL_MODS_FILELIST) $(TOP_BB_MODS_FILELIST) $(MODEL_BB_MODS_FILELIST) &: $(BB_MODS_FILELIST) $(MFC_TOP_HRCHY_JSON) $(FINAL_ANNO_FILE) diff --git a/scripts/split-module-files.py b/scripts/split-module-files.py deleted file mode 100755 index 64fe570704..0000000000 --- a/scripts/split-module-files.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 - -import json -import argparse -from typing import List, Optional - -# Schema of json emitted by circt -""" -{ - "instance_name": "TestHarness", - "module_name": "TestHarness", - "instances": [ - { - "instance_name": "chiptop", - "module_name": "ChipTop", - "instances": [ - { - "instance_name": "system", - "module_name": "DigitalTop", - "instances": [ ] - }, ... - ] - }, - { - "instance_name": "simdram", - "module_name": "SimDRAM", - "instances": [] - }, - ] -} -""" - -def get_modules(js: dict) -> List[str]: - if 'instances' not in js: - return js['module_name'] - else: - mods = [] - for mod in js['instances']: - mods.extend(get_modules(mod)) - return [js['module_name']] + mods - -def find_mod_by_name(js: dict, name: str) -> Optional[List[dict]]: - if 'instances' not in js: - return None - else: - mods = [] - for mod in js['instances']: - if mod['module_name'] == name: - mods.append(mod) - other_mods = find_mod_by_name(mod, name) - if other_mods is not None: - mods.extend(other_mods) - return mods - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Convert CIRCT (firtool) hierarchy JSON into DUT and test harness filelists') - parser.add_argument('--model-hier-json', type=str, required=True, help='Path to hierarchy JSON emitted by firtool. Must include DUT as a module.') - parser.add_argument('--dut', type=str, required=True, help='Name of the DUT module.') - parser.add_argument('--out-dut-filelist', type=str, required=True, help='Path to output filelist including all modules under the DUT.') - parser.add_argument('--out-model-filelist', type=str, required=True, help='Path to output filelist including all modules under the top-most module but not modules under the DUT.') - parser.add_argument('--in-all-filelist', type=str, required=True, help='Path to input filelist that has all modules (relative paths).') - parser.add_argument('--target-dir', type=str, required=True, help='Path to where module sources are located (combined with --in-all-filelist gives the absolute path to module sources).') - args = parser.parse_args() - - with open(args.model_hier_json) as f: - j = json.load(f) - - dut_tops = find_mod_by_name(j, args.dut) - assert dut_tops is not None - assert len(dut_tops) == 1 - dut_top = dut_tops[0] - - dut_mods = set(get_modules(dut_top)) - model_mods = set(get_modules(j)) - dut_mods - both_mods = dut_mods.intersection(model_mods) - - assert len(both_mods) == 0 - - with open(args.out_dut_filelist, 'w') as df, \ - open(args.in_all_filelist) as fl: - # add paths that correspond to modules to output file - for path in fl: - writeOut = False - for dm in dut_mods: - if dm in path: - writeOut = True - break - - # prepend the target directory to get filelist with absolute paths - if writeOut: - if not args.target_dir in path: - df.write(f"{args.target_dir}/{path}") - else: - df.write(f"{path}") - - with open(args.out_model_filelist, 'w') as df, \ - open(args.in_all_filelist) as fl: - # add paths that correspond to modules to output file - for path in fl: - writeOut = False - for dm in model_mods: - if dm in path: - writeOut = True - break - - # prepend the target directory to get filelist with absolute paths - if writeOut: - if not args.target_dir in path: - df.write(f"{args.target_dir}/{path}") - else: - df.write(f"{path}") diff --git a/scripts/uniqify-module-names.py b/scripts/uniqify-module-names.py index a6255a8c90..87b56b63bf 100755 --- a/scripts/uniqify-module-names.py +++ b/scripts/uniqify-module-names.py @@ -5,19 +5,21 @@ import shutil import os import datetime - +import sys parser = argparse.ArgumentParser(description="") -parser.add_argument("--top-filelist", type=str, required=True, help="Abs path to ..top.f") -parser.add_argument("--mod-filelist", type=str, required=True, help="Abs path to ..model.f") -parser.add_argument("--gen-collateral-path", dest="gcpath", type=str, required=True, help="Abs path to the gen-collateral directory") parser.add_argument("--model-hier-json", type=str, required=True, help="Path to hierarchy JSON emitted by firtool. Must include DUT as a module.") -parser.add_argument("--out-model-hier-json", type=str, required=True, help="Path to updated hierarchy JSON emitted by this script.") +parser.add_argument("--top-hier-json", type=str, required=True, help="Path to hierarchy JSON emitted by firtool. Must include DUT as a module.") +parser.add_argument('--in-all-filelist', type=str, required=True, help='Path to input filelist that has all modules (relative paths).') parser.add_argument("--dut", type=str, required=True, help="Name of the DUT module.") parser.add_argument("--model", type=str, required=True, help="Name of the Model module.") +parser.add_argument('--target-dir', type=str, required=True, help='Path to where module sources are located (combined with --in-all-filelist gives the absolute path to module sources).') +parser.add_argument('--out-dut-filelist', type=str, required=True, help='Path to output filelist including all modules under the DUT.') +parser.add_argument('--out-model-filelist', type=str, required=True, help='Path to output filelist including all modules under the MODEL.') +parser.add_argument("--out-model-hier-json", type=str, required=True, help="Path to updated hierarchy JSON emitted by this script.") +parser.add_argument("--gcpath", type=str, required=True, help="Path to gen-collateral") args = parser.parse_args() - MODEL_SFX=args.model + "_UNIQUIFIED" def bash(cmd): @@ -40,9 +42,6 @@ def get_filelist(filelist): print(f"Something is wrong about this line '{line}'") return fnames -def update_filelist(cur_file, new_file): - bash(f"echo \"{args.gcpath}/{new_file}\" >> {os.path.join(args.gcpath, args.mod_filelist)}") - def generate_copy(c, sfx): (cur_name, ext) = os.path.splitext(c) new_name = cur_name + "_" + sfx @@ -55,46 +54,167 @@ def generate_copy(c, sfx): bash(f"sed -i s/\"module {cur_name}\"/\"module {new_name}\"/ {new_file}") return new_file -def dfs_update_modules(tree, common_fnames, visited, top_fnames): +def dfs_update_modules(tree, common_fnames, visited, ext_dict): # List of direct submodules to update childs_to_update = list() for child in tree['instances']: # We don't have to change stuff that are under the dut - if (child['module_name'] == args.dut) or (child['module_name'] in visited): + if (child['module_name'] == args.dut): continue - if dfs_update_modules(child, common_fnames, visited, top_fnames): + if dfs_update_modules(child, common_fnames, visited, ext_dict): childs_to_update.append(child['module_name']) - if (child['module_name'] + ".sv") in common_fnames: + if (child['module_name']) in common_fnames: child['module_name'] = child['module_name'] + "_" + MODEL_SFX cur_module = tree['module_name'] - cur_file = cur_module + ".sv" new_file = None # cur_file is in the common list, or is a ancestor of of them, generate a new file - if (cur_file in common_fnames) or len(childs_to_update) > 0: - new_file = generate_copy(cur_file, MODEL_SFX) - update_filelist(cur_file, os.path.basename(new_file)) - - for submodule_name in childs_to_update: - if (submodule_name + ".sv") in common_fnames: - bash(f"sed -i s/\"{submodule_name}\"/\"{submodule_name}_{MODEL_SFX}\"/ {new_file}") + if (cur_module in common_fnames) or len(childs_to_update) > 0: + new_file = 1 visited.add(cur_module) return (new_file is not None) +def bfs_update(tree, common_fnames, ext_dict, filelist): + q = [(tree['instance_name'], tree['module_name'], tree['instances'], None)] + + updated_submodule = set() + + while len(q) != 0: + front = q[0] + q.pop(0) + (inst, mod, child, parent) = front + + try: + cur_file = mod + "." + ext_dict[mod] + except: + cur_file = mod + ".sv" + + mod_updated = False + + # if the module is common, make a copy & update its instance in its parent + if mod in common_fnames: + mod_updated = True + new_file = generate_copy(cur_file, MODEL_SFX) + filelist.append(new_file) + if parent is not None and ((parent, mod) not in updated_submodule): + print(mod, parent) + parent_file = os.path.join(args.gcpath, parent + "." + ext_dict[parent]) + bash(f"sed -i s/\"{mod}\"/\"{mod}_{MODEL_SFX}\"/ {parent_file}") + updated_submodule.add((parent, mod)) + else: + filelist.append(cur_file) + + # set the parent module name + new_mod = mod + if mod_updated: + new_mod = mod + "_" + MODEL_SFX + ext_dict[new_mod] = ext_dict[mod] + + # traverse its children + for c in child: + if c['module_name'] != args.dut: + q.append((c['instance_name'], c['module_name'], c['instances'], new_mod)) + +def bfs_collect_modules(tree, child_to_ignore = None): + q = [(tree['instance_name'], tree['module_name'], tree['instances'])] + + modules = list() + while len(q) != 0: + front = q[0] + q.pop(0) + + (inst, mod, child) = front + modules.append(mod) + for c in child: + if c['module_name'] != child_to_ignore: + print(c['module_name']) + q.append((c['instance_name'], c['module_name'], c['instances'])) + return modules + +def write_filelist(modules, out_file): + with open(out_file, "w") as df, \ + open(args.in_all_filelist) as fl: + # add paths that correspond to modules to output file + for path in fl: + writeOut = False + for dm in modules: + if dm in path: + writeOut = True + break + + # prepend the target directory to get filelist with absolute paths + if writeOut: + if not args.target_dir in path: + df.write(f"{args.target_dir}/{path}") + else: + df.write(f"{path}") + +def write_filelist_model(modules, out_file): + with open(out_file, "w") as df: + for m in modules: + if not args.target_dir in m: + df.write(f"{args.target_dir}/{m}\n") + else: + df.write(f"{m}\n") + +def get_file_ext(all_filelist): + ext_dict = dict() + with open(args.in_all_filelist) as fl: + for path in fl: + fname = os.path.basename(path) + (module, ext) = fname.strip().split(".") + ext_dict[module] = ext + return ext_dict + def main(): - top_fnames = set(get_filelist(args.top_filelist)) - mod_fnames = set(get_filelist(args.mod_filelist)) - common_fnames = top_fnames.intersection(mod_fnames) + with open(args.model_hier_json) as imhj: + imhj_data = json.load(imhj) + modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) + + for x in modules_under_model: + print(f"model only {x}") + + with open(args.top_hier_json) as imhj: + imhj_data = json.load(imhj) + modules_under_top = set(bfs_collect_modules(imhj_data)) + + for x in modules_under_top: + print(f"top only {x}") + + common_modules = modules_under_top.intersection(modules_under_model) + + print(f"modules under top {len(modules_under_top)}") + print(f"modules under model {len(modules_under_model)}") + print(f"modules under both {len(common_modules)}") + print(f"total modules {len(modules_under_top) + len(modules_under_model) - len(common_modules)}") + + write_filelist(modules_under_top, args.out_dut_filelist) + + ext_dict = get_file_ext(args.in_all_filelist) + print(f"total modules in filelist {len(ext_dict)}") + + for x in common_modules: + print(f"common {x}") + +# for x in common_fnames: +# print(f"common_fnames {x}") + with open(args.model_hier_json) as imhj: imhj_data = json.load(imhj) with open(args.out_model_hier_json, "w+") as out_file: visited = set() - dfs_update_modules(imhj_data, common_fnames, visited, top_fnames) + filelist = list() + bfs_update(imhj_data, common_modules, ext_dict, filelist) + dfs_update_modules(imhj_data, common_modules, visited, ext_dict) json.dump(imhj_data, out_file, indent=2) + updated_modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) + print(filelist) + write_filelist_model(set(filelist), args.out_model_filelist) + if __name__ == "__main__": main() From 85abce536a4ff7e93e8cfde545d1c3cdffb981d3 Mon Sep 17 00:00:00 2001 From: joey0320 Date: Thu, 27 Apr 2023 20:42:34 -0700 Subject: [PATCH 02/10] cleanup --- scripts/uniqify-module-names.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/scripts/uniqify-module-names.py b/scripts/uniqify-module-names.py index 87b56b63bf..28d5f971cc 100755 --- a/scripts/uniqify-module-names.py +++ b/scripts/uniqify-module-names.py @@ -129,7 +129,6 @@ def bfs_collect_modules(tree, child_to_ignore = None): modules.append(mod) for c in child: if c['module_name'] != child_to_ignore: - print(c['module_name']) q.append((c['instance_name'], c['module_name'], c['instances'])) return modules @@ -173,34 +172,13 @@ def main(): imhj_data = json.load(imhj) modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) - for x in modules_under_model: - print(f"model only {x}") - with open(args.top_hier_json) as imhj: imhj_data = json.load(imhj) modules_under_top = set(bfs_collect_modules(imhj_data)) - for x in modules_under_top: - print(f"top only {x}") - common_modules = modules_under_top.intersection(modules_under_model) - - print(f"modules under top {len(modules_under_top)}") - print(f"modules under model {len(modules_under_model)}") - print(f"modules under both {len(common_modules)}") - print(f"total modules {len(modules_under_top) + len(modules_under_model) - len(common_modules)}") - write_filelist(modules_under_top, args.out_dut_filelist) - ext_dict = get_file_ext(args.in_all_filelist) - print(f"total modules in filelist {len(ext_dict)}") - - for x in common_modules: - print(f"common {x}") - -# for x in common_fnames: -# print(f"common_fnames {x}") - with open(args.model_hier_json) as imhj: imhj_data = json.load(imhj) @@ -213,7 +191,6 @@ def main(): json.dump(imhj_data, out_file, indent=2) updated_modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) - print(filelist) write_filelist_model(set(filelist), args.out_model_filelist) if __name__ == "__main__": From ab9ec1650e5fc0e24295e165e0625c31c791d2b5 Mon Sep 17 00:00:00 2001 From: joey0320 Date: Fri, 28 Apr 2023 09:15:33 -0700 Subject: [PATCH 03/10] fixes --- common.mk | 2 +- ...dule-names.py => uniquify-module-names.py} | 31 ++++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) rename scripts/{uniqify-module-names.py => uniquify-module-names.py} (91%) diff --git a/common.mk b/common.mk index 4aeb44aa2d..7afdf57b05 100644 --- a/common.mk +++ b/common.mk @@ -232,7 +232,7 @@ $(SFC_MFC_TARGETS) &: $(FIRRTL_FILE) $(FINAL_ANNO_FILE) $(SFC_LEVEL) $(EXTRA_FIR # DOC include end: FirrtlCompiler $(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(ALL_MODS_FILELIST) $(BB_MODS_FILELIST) $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) &: $(MFC_MODEL_HRCHY_JSON) $(MFC_TOP_HRCHY_JSON) $(MFC_FILELIST) $(MFC_BB_MODS_FILELIST) - $(base_dir)/scripts/uniqify-module-names.py \ + $(base_dir)/scripts/uniquify-module-names.py \ --model-hier-json $(MFC_MODEL_HRCHY_JSON) \ --top-hier-json $(MFC_TOP_HRCHY_JSON) \ --in-all-filelist $(MFC_FILELIST) \ diff --git a/scripts/uniqify-module-names.py b/scripts/uniquify-module-names.py similarity index 91% rename from scripts/uniqify-module-names.py rename to scripts/uniquify-module-names.py index 28d5f971cc..7eb03a7c52 100755 --- a/scripts/uniqify-module-names.py +++ b/scripts/uniquify-module-names.py @@ -97,14 +97,13 @@ def bfs_update(tree, common_fnames, ext_dict, filelist): if mod in common_fnames: mod_updated = True new_file = generate_copy(cur_file, MODEL_SFX) - filelist.append(new_file) + filelist.append((mod, new_file)) if parent is not None and ((parent, mod) not in updated_submodule): - print(mod, parent) parent_file = os.path.join(args.gcpath, parent + "." + ext_dict[parent]) - bash(f"sed -i s/\"{mod}\"/\"{mod}_{MODEL_SFX}\"/ {parent_file}") + bash(f"sed -i s/\"{mod} \"/\"{mod}_{MODEL_SFX} \"/ {parent_file}") updated_submodule.add((parent, mod)) else: - filelist.append(cur_file) + filelist.append((mod, cur_file)) # set the parent module name new_mod = mod @@ -150,20 +149,24 @@ def write_filelist(modules, out_file): else: df.write(f"{path}") -def write_filelist_model(modules, out_file): +def write_filelist_model(modules, out_file, ext_dict): with open(out_file, "w") as df: - for m in modules: - if not args.target_dir in m: - df.write(f"{args.target_dir}/{m}\n") - else: - df.write(f"{m}\n") + for (m, fname) in modules: + if m in ext_dict.keys(): + if not args.target_dir in fname: + df.write(f"{args.target_dir}/{fname}\n") + else: + df.write(f"{fname}\n") def get_file_ext(all_filelist): ext_dict = dict() - with open(args.in_all_filelist) as fl: + with open(all_filelist) as fl: for path in fl: fname = os.path.basename(path) - (module, ext) = fname.strip().split(".") + fname_strip = fname.strip().split(".") + ext = fname_strip[-1] + fname_strip.pop() + module = ".".join(fname_strip) ext_dict[module] = ext return ext_dict @@ -189,9 +192,7 @@ def main(): bfs_update(imhj_data, common_modules, ext_dict, filelist) dfs_update_modules(imhj_data, common_modules, visited, ext_dict) json.dump(imhj_data, out_file, indent=2) - - updated_modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) - write_filelist_model(set(filelist), args.out_model_filelist) + write_filelist_model(set(filelist), args.out_model_filelist, ext_dict) if __name__ == "__main__": main() From 27728a70efa3ab7a5acb4133939687d80ea46b8a Mon Sep 17 00:00:00 2001 From: joey0320 Date: Fri, 28 Apr 2023 12:27:54 -0700 Subject: [PATCH 04/10] fix for ext modules --- scripts/uniquify-module-names.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/uniquify-module-names.py b/scripts/uniquify-module-names.py index 7eb03a7c52..5b058651c8 100755 --- a/scripts/uniquify-module-names.py +++ b/scripts/uniquify-module-names.py @@ -95,13 +95,16 @@ def bfs_update(tree, common_fnames, ext_dict, filelist): # if the module is common, make a copy & update its instance in its parent if mod in common_fnames: - mod_updated = True - new_file = generate_copy(cur_file, MODEL_SFX) - filelist.append((mod, new_file)) - if parent is not None and ((parent, mod) not in updated_submodule): - parent_file = os.path.join(args.gcpath, parent + "." + ext_dict[parent]) - bash(f"sed -i s/\"{mod} \"/\"{mod}_{MODEL_SFX} \"/ {parent_file}") - updated_submodule.add((parent, mod)) + try: + new_file = generate_copy(cur_file, MODEL_SFX) + filelist.append((mod, new_file)) + if parent is not None and ((parent, mod) not in updated_submodule): + parent_file = os.path.join(args.gcpath, parent + "." + ext_dict[parent]) + bash(f"sed -i s/\"{mod} \"/\"{mod}_{MODEL_SFX} \"/ {parent_file}") + updated_submodule.add((parent, mod)) + mod_updated = True + except: + print(f"No corresponding file for {cur_file}") else: filelist.append((mod, cur_file)) From 30ec9806e03c4d0f72c05f7d5f5754a662f757e1 Mon Sep 17 00:00:00 2001 From: joey0320 Date: Fri, 28 Apr 2023 13:30:34 -0700 Subject: [PATCH 05/10] final fix --- scripts/uniquify-module-names.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/uniquify-module-names.py b/scripts/uniquify-module-names.py index 5b058651c8..641ef56367 100755 --- a/scripts/uniquify-module-names.py +++ b/scripts/uniquify-module-names.py @@ -141,7 +141,11 @@ def write_filelist(modules, out_file): for path in fl: writeOut = False for dm in modules: - if dm in path: + bm_ext = os.path.basename(path).split(".") + bm_ext.pop() + bm = ".".join(bm_ext) + print(bm) + if dm == bm: writeOut = True break From 3f80507ce4a2f8ae659da2d27ab81d0a3be8f564 Mon Sep 17 00:00:00 2001 From: joey0320 Date: Sat, 29 Apr 2023 18:21:48 -0700 Subject: [PATCH 06/10] rm split-bb-files.py --- common.mk | 10 +--- scripts/split-bb-files.py | 82 -------------------------------- scripts/uniquify-module-names.py | 18 ++++--- variables.mk | 4 -- vlsi/Makefile | 4 +- 5 files changed, 15 insertions(+), 103 deletions(-) delete mode 100755 scripts/split-bb-files.py diff --git a/common.mk b/common.mk index 7afdf57b05..0868088b3c 100644 --- a/common.mk +++ b/common.mk @@ -249,14 +249,6 @@ $(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(ALL_MODS_FILELIST) $(BB_MODS_FILEL $(SED) -i 's/\.\///' $(BB_MODS_FILELIST) sort -u $(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(BB_MODS_FILELIST) > $(ALL_MODS_FILELIST) -$(TOP_BB_MODS_FILELIST) $(MODEL_BB_MODS_FILELIST) &: $(BB_MODS_FILELIST) $(MFC_TOP_HRCHY_JSON) $(FINAL_ANNO_FILE) - $(base_dir)/scripts/split-bb-files.py \ - --in-bb-f $(BB_MODS_FILELIST) \ - --in-top-hrchy-json $(MFC_TOP_HRCHY_JSON) \ - --in-anno-json $(FINAL_ANNO_FILE) \ - --out-top-bb-f $(TOP_BB_MODS_FILELIST) \ - --out-model-bb-f $(MODEL_BB_MODS_FILELIST) - $(TOP_SMEMS_CONF) $(MODEL_SMEMS_CONF) &: $(MFC_SMEMS_CONF) $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) $(base_dir)/scripts/split-mems-conf.py \ --in-smems-conf $(MFC_SMEMS_CONF) \ @@ -280,7 +272,7 @@ $(MODEL_SMEMS_FILE) $(MODEL_SMEMS_FIR) &: $(MODEL_SMEMS_CONF) | $(TOP_SMEMS_FILE # note: {MODEL,TOP}_BB_MODS_FILELIST is added as a req. so that the files get generated, # however it is really unneeded since ALL_MODS_FILELIST includes all BB files ######################################################################################## -$(sim_common_files): $(sim_files) $(ALL_MODS_FILELIST) $(TOP_SMEMS_FILE) $(MODEL_SMEMS_FILE) $(TOP_BB_MODS_FILELIST) $(MODEL_BB_MODS_FILELIST) +$(sim_common_files): $(sim_files) $(ALL_MODS_FILELIST) $(TOP_SMEMS_FILE) $(MODEL_SMEMS_FILE) $(BB_MODS_FILELIST) sort -u $(sim_files) $(ALL_MODS_FILELIST) | grep -v '.*\.\(svh\|h\)$$' > $@ echo "$(TOP_SMEMS_FILE)" >> $@ echo "$(MODEL_SMEMS_FILE)" >> $@ diff --git a/scripts/split-bb-files.py b/scripts/split-bb-files.py deleted file mode 100755 index 959a10a2e5..0000000000 --- a/scripts/split-bb-files.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 - -import json -import argparse -from collections import defaultdict - -# Schema of *.f emitted by circt -""" -//gen-collateral/SimUART.cc -//gen-collateral/AsyncQueueSource.sv -//gen-collateral/AsyncQueueSink.sv -//gen-collateral/AsyncQueueSource_1.sv -//gen-collateral/AsyncQueueSink_1.sv -//gen-collateral/AsyncQueueSource_2.sv -//gen-collateral/AsyncQueueSink_2.sv -//gen-collateral/AsyncResetSynchronizerShiftReg_w4_d3_i0.sv -""" - -def bfs_collect_submodules(tree): - output = set() - q = [(tree['instance_name'], tree['module_name'], tree['instances'])] - - while len(q) != 0: - front = q[0] - q.pop(0) - - (inst, mod, child) = front - output.add(mod) - for c in child: - q.append((c['instance_name'], c['module_name'], c['instances'])) - return output - -def write_lines_to_file(lines, file_path): - with open(file_path, "w") as fp: - for line in lines: - fp.write("%s\n" % line) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Create *.model.bb.f and *.top.bb.f blackbox filelists') - parser.add_argument('--in-bb-f', type=str, required=True, help='All blackbox files filelist (includes both MODEL/TOP files)') - parser.add_argument('--in-top-hrchy-json', type=str, required=True, help='List containing hierarchy of top modules (top-module-hierarchy.json)') - parser.add_argument('--in-anno-json', type=str, required=True, help='Anno. file with blackbox annotations') - parser.add_argument('--out-top-bb-f', type=str, required=True, help='List of blackbox files for TOP') - parser.add_argument('--out-model-bb-f', type=str, required=True, help='List of blackbox files for MODEL') - args = parser.parse_args() - - # module_path -> list of bb paths (not fully resolved paths) - mod_bb_dict = defaultdict(list) - with open(args.in_anno_json, "r") as f: - anno_data = json.load(f) - for anno in anno_data: - if 'BlackBoxInlineAnno' in anno['class']: - mod_bb_dict[anno['target']].append(anno['name']) - if 'BlackBoxPathAnno' in anno['class']: - mod_bb_dict[anno['target']].append(anno['path']) - - with open(args.in_top_hrchy_json) as ihj: - ihj_data = json.load(ihj) - top_inner_modules = bfs_collect_submodules(ihj_data) - - with open(args.in_bb_f) as ibf: - lines = ibf.read().splitlines() - - tbfs = set() - for mod_path, bb_files in mod_bb_dict.items(): - leaf_mod = mod_path.split('.')[-1] - - # if matched, add the fully resolved path to the top bb filelist - if leaf_mod in top_inner_modules: - for line in lines: - for bb_file in bb_files: - if bb_file in line: - tbfs.add(line) - - # now tbfs should be complete (need to remove tbf files from original bb file for model bb) - mbfs = set() - for line in lines: - if not line in tbfs: - mbfs.add(line) - - write_lines_to_file(tbfs, args.out_top_bb_f) - write_lines_to_file(mbfs, args.out_model_bb_f) diff --git a/scripts/uniquify-module-names.py b/scripts/uniquify-module-names.py index 641ef56367..768a061e60 100755 --- a/scripts/uniquify-module-names.py +++ b/scripts/uniquify-module-names.py @@ -54,14 +54,14 @@ def generate_copy(c, sfx): bash(f"sed -i s/\"module {cur_name}\"/\"module {new_name}\"/ {new_file}") return new_file -def dfs_update_modules(tree, common_fnames, visited, ext_dict): +def dfs_update_modules(tree, common_fnames, visited): # List of direct submodules to update childs_to_update = list() for child in tree['instances']: # We don't have to change stuff that are under the dut if (child['module_name'] == args.dut): continue - if dfs_update_modules(child, common_fnames, visited, ext_dict): + if dfs_update_modules(child, common_fnames, visited): childs_to_update.append(child['module_name']) if (child['module_name']) in common_fnames: child['module_name'] = child['module_name'] + "_" + MODEL_SFX @@ -87,7 +87,7 @@ def bfs_update(tree, common_fnames, ext_dict, filelist): (inst, mod, child, parent) = front try: - cur_file = mod + "." + ext_dict[mod] + cur_file = mod + "." + ext_dict[mod][0] except: cur_file = mod + ".sv" @@ -99,7 +99,7 @@ def bfs_update(tree, common_fnames, ext_dict, filelist): new_file = generate_copy(cur_file, MODEL_SFX) filelist.append((mod, new_file)) if parent is not None and ((parent, mod) not in updated_submodule): - parent_file = os.path.join(args.gcpath, parent + "." + ext_dict[parent]) + parent_file = os.path.join(args.gcpath, parent + "." + ext_dict[parent][0]) bash(f"sed -i s/\"{mod} \"/\"{mod}_{MODEL_SFX} \"/ {parent_file}") updated_submodule.add((parent, mod)) mod_updated = True @@ -165,6 +165,10 @@ def write_filelist_model(modules, out_file, ext_dict): else: df.write(f"{fname}\n") + if len(ext_dict[m]) > 1: + assert(len(ext_dict[m]) == 2) + df.write(f"{args.target_dir}/{m}.{ext_dict[m][1]}\n") + def get_file_ext(all_filelist): ext_dict = dict() with open(all_filelist) as fl: @@ -174,7 +178,9 @@ def get_file_ext(all_filelist): ext = fname_strip[-1] fname_strip.pop() module = ".".join(fname_strip) - ext_dict[module] = ext + if module not in ext_dict.keys(): + ext_dict[module] = list() + ext_dict[module].append(ext) return ext_dict def main(): @@ -197,7 +203,7 @@ def main(): visited = set() filelist = list() bfs_update(imhj_data, common_modules, ext_dict, filelist) - dfs_update_modules(imhj_data, common_modules, visited, ext_dict) + dfs_update_modules(imhj_data, common_modules, visited) json.dump(imhj_data, out_file, indent=2) write_filelist_model(set(filelist), args.out_model_filelist, ext_dict) diff --git a/variables.mk b/variables.mk index 1f659ac7ca..443caa94a9 100644 --- a/variables.mk +++ b/variables.mk @@ -186,10 +186,6 @@ MODEL_MODS_FILELIST ?= $(build_dir)/$(long_name).model.f # list of all blackbox files (may be included in the top/model.f files) # this has the build_dir appended BB_MODS_FILELIST ?= $(build_dir)/$(long_name).bb.f -# top blackbox module files to include -TOP_BB_MODS_FILELIST ?= $(build_dir)/$(long_name).top.bb.f -# model blackbox module files to include (not including top blackbox modules) -MODEL_BB_MODS_FILELIST ?= $(build_dir)/$(long_name).model.bb.f # all module files to include (top, model, bb included) ALL_MODS_FILELIST ?= $(build_dir)/$(long_name).all.f diff --git a/vlsi/Makefile b/vlsi/Makefile index 8259e7a6ec..ad51c75443 100644 --- a/vlsi/Makefile +++ b/vlsi/Makefile @@ -72,7 +72,7 @@ VLSI_RTL = $(build_dir)/syn.f ifneq ($(CUSTOM_VLOG), ) RTL_DEPS = $(CUSTOM_VLOG) else - RTL_DEPS = $(TOP_MODS_FILELIST) $(TOP_BB_MODS_FILELIST) $(TOP_SMEMS_FILE) + RTL_DEPS = $(TOP_MODS_FILELIST) $(TOP_SMEMS_FILE) endif $(VLSI_RTL): $(RTL_DEPS) @@ -80,7 +80,7 @@ ifneq ($(CUSTOM_VLOG), ) > $(VLSI_RTL) $(foreach file,$^,echo $(file) >> $(VLSI_RTL)) else - cat $(TOP_MODS_FILELIST) $(TOP_BB_MODS_FILELIST) | sort -u > $(VLSI_RTL) + cat $(TOP_MODS_FILELIST) | sort -u > $(VLSI_RTL) echo $(TOP_SMEMS_FILE) >> $(VLSI_RTL) endif From a5bf60c0f1cdb3376559b887ffcfdda2109e8f38 Mon Sep 17 00:00:00 2001 From: joey0320 Date: Sat, 29 Apr 2023 21:32:00 -0700 Subject: [PATCH 07/10] oops --- vlsi/sim.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vlsi/sim.mk b/vlsi/sim.mk index 13a7fcc3e0..291c93adf6 100644 --- a/vlsi/sim.mk +++ b/vlsi/sim.mk @@ -10,7 +10,7 @@ $(SIM_CONF): $(sim_common_files) echo " top_module: $(VLSI_TOP)" >> $@ echo " tb_name: ''" >> $@ # don't specify -top echo " input_files:" >> $@ - for x in $$(cat $(MODEL_MODS_FILELIST) $(MODEL_BB_MODS_FILELIST) | sort -u) $(MODEL_SMEMS_FILE) $(SIM_FILE_REQS); do \ + for x in $$(cat $(MODEL_MODS_FILELIST) | sort -u) $(MODEL_SMEMS_FILE) $(SIM_FILE_REQS); do \ echo ' - "'$$x'"' >> $@; \ done echo " input_files_meta: 'append'" >> $@ From 884cf3cf6128cbe910a03d8913d10f2448db83e1 Mon Sep 17 00:00:00 2001 From: joey0320 Date: Sun, 30 Apr 2023 13:32:46 -0700 Subject: [PATCH 08/10] deal with bb --- scripts/uniquify-module-names.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/scripts/uniquify-module-names.py b/scripts/uniquify-module-names.py index 768a061e60..bd7ddcd992 100755 --- a/scripts/uniquify-module-names.py +++ b/scripts/uniquify-module-names.py @@ -134,7 +134,7 @@ def bfs_collect_modules(tree, child_to_ignore = None): q.append((c['instance_name'], c['module_name'], c['instances'])) return modules -def write_filelist(modules, out_file): +def write_filelist(modules, out_file, files_written): with open(out_file, "w") as df, \ open(args.in_all_filelist) as fl: # add paths that correspond to modules to output file @@ -144,19 +144,20 @@ def write_filelist(modules, out_file): bm_ext = os.path.basename(path).split(".") bm_ext.pop() bm = ".".join(bm_ext) - print(bm) - if dm == bm: + print(dm, bm, bm_ext) + if (dm == bm) or (dm == bm_ext[0]): writeOut = True break # prepend the target directory to get filelist with absolute paths if writeOut: + files_written.add(os.path.basename(path)) if not args.target_dir in path: df.write(f"{args.target_dir}/{path}") else: df.write(f"{path}") -def write_filelist_model(modules, out_file, ext_dict): +def write_filelist_model(modules, out_file, ext_dict, files_written): with open(out_file, "w") as df: for (m, fname) in modules: if m in ext_dict.keys(): @@ -164,10 +165,7 @@ def write_filelist_model(modules, out_file, ext_dict): df.write(f"{args.target_dir}/{fname}\n") else: df.write(f"{fname}\n") - - if len(ext_dict[m]) > 1: - assert(len(ext_dict[m]) == 2) - df.write(f"{args.target_dir}/{m}.{ext_dict[m][1]}\n") + files_written.add(os.path.basename(fname)) def get_file_ext(all_filelist): ext_dict = dict() @@ -188,12 +186,15 @@ def main(): imhj_data = json.load(imhj) modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) + + files_written = set() + with open(args.top_hier_json) as imhj: imhj_data = json.load(imhj) modules_under_top = set(bfs_collect_modules(imhj_data)) common_modules = modules_under_top.intersection(modules_under_model) - write_filelist(modules_under_top, args.out_dut_filelist) + write_filelist(modules_under_top, args.out_dut_filelist, files_written) ext_dict = get_file_ext(args.in_all_filelist) with open(args.model_hier_json) as imhj: @@ -205,7 +206,15 @@ def main(): bfs_update(imhj_data, common_modules, ext_dict, filelist) dfs_update_modules(imhj_data, common_modules, visited) json.dump(imhj_data, out_file, indent=2) - write_filelist_model(set(filelist), args.out_model_filelist, ext_dict) + write_filelist_model(set(filelist), args.out_model_filelist, ext_dict, files_written) + + with open(args.out_model_filelist, "a") as of, \ + open(args.in_all_filelist) as fl: + for path in fl: + fname = os.path.basename(path) + ext = fname.strip().split(".")[-1] + if (fname not in files_written) and (ext == "cc"): + of.write(f"{args.target_dir}/{fname}") if __name__ == "__main__": main() From 60e80a772ef59af8db7c1ccd8502ce9fd69acdfb Mon Sep 17 00:00:00 2001 From: joey0320 Date: Thu, 4 May 2023 20:15:21 -0700 Subject: [PATCH 09/10] Cleanup + fixes, think it's stable now --- scripts/uniquify-module-names.py | 259 +++++++++++++++---------------- 1 file changed, 123 insertions(+), 136 deletions(-) diff --git a/scripts/uniquify-module-names.py b/scripts/uniquify-module-names.py index bd7ddcd992..12a0843e54 100755 --- a/scripts/uniquify-module-names.py +++ b/scripts/uniquify-module-names.py @@ -4,24 +4,25 @@ import argparse import shutil import os -import datetime import sys + parser = argparse.ArgumentParser(description="") parser.add_argument("--model-hier-json", type=str, required=True, help="Path to hierarchy JSON emitted by firtool. Must include DUT as a module.") parser.add_argument("--top-hier-json", type=str, required=True, help="Path to hierarchy JSON emitted by firtool. Must include DUT as a module.") parser.add_argument('--in-all-filelist', type=str, required=True, help='Path to input filelist that has all modules (relative paths).') parser.add_argument("--dut", type=str, required=True, help="Name of the DUT module.") parser.add_argument("--model", type=str, required=True, help="Name of the Model module.") -parser.add_argument('--target-dir', type=str, required=True, help='Path to where module sources are located (combined with --in-all-filelist gives the absolute path to module sources).') parser.add_argument('--out-dut-filelist', type=str, required=True, help='Path to output filelist including all modules under the DUT.') parser.add_argument('--out-model-filelist', type=str, required=True, help='Path to output filelist including all modules under the MODEL.') parser.add_argument("--out-model-hier-json", type=str, required=True, help="Path to updated hierarchy JSON emitted by this script.") +parser.add_argument('--target-dir', type=str, required=True, help='Path to where module sources are located (combined with --in-all-filelist gives the absolute path to module sources).') parser.add_argument("--gcpath", type=str, required=True, help="Path to gen-collateral") args = parser.parse_args() MODEL_SFX=args.model + "_UNIQUIFIED" + def bash(cmd): fail = os.system(cmd) if fail: @@ -30,17 +31,74 @@ def bash(cmd): else: print(cmd) -def get_filelist(filelist): - fnames = [] - with open(filelist) as f: +def bfs_collect_modules(tree, child_to_ignore = None): + q = [(tree['instance_name'], tree['module_name'], tree['instances'])] + + modules = list() + while len(q) != 0: + front = q[0] + q.pop(0) + + (inst, mod, child) = front + modules.append(mod) + for c in child: + if c['module_name'] != child_to_ignore: + q.append((c['instance_name'], c['module_name'], c['instances'])) + return modules + +def get_modules_in_verilog_file(file): + module_names = list() + with open(file) as f: lines = f.readlines() for line in lines: - try: - fname = line.split("/")[-1].strip() - fnames.append(fname) - except: - print(f"Something is wrong about this line '{line}'") - return fnames + words = line.split() + if len(words) > 0 and words[0] == "module": + module_names.append(words[1].replace("(", "")) + return module_names + +def get_modules_in_filelist(verilog_module_filename, cc_filelist): + with open(args.in_all_filelist) as fl: + lines = fl.readlines() + for line in lines: + path = line.strip() + basepath = os.path.basename(path) + ext = basepath.split(".")[-1] + + if (ext == "v") or (ext == "sv"): + modules = get_modules_in_verilog_file(os.path.join(args.gcpath, basepath)) + for module in modules: + verilog_module_filename[module] = basepath + else: + cc_filelist.append(basepath) + return (verilog_module_filename, cc_filelist) + +def get_modules_under_hier(hier, child_to_ignore=None): + with open(hier) as hj: + hj_data = json.load(hj) + modules_under_hier = set(bfs_collect_modules(hj_data, child_to_ignore=child_to_ignore)) + return modules_under_hier + +def write_verilog_filelist(modules, verilog_module_filename, out_filelist): + written_files = set() + existing_modules = verilog_module_filename.keys() + + with open(out_filelist, "w") as df: + for module in modules: + if module in existing_modules: + verilog_filename = verilog_module_filename[module] + if verilog_filename not in written_files: + written_files.add(verilog_filename) + if args.target_dir in verilog_filename: + df.write(f"{verilog_filename}\n") + else: + df.write(f"{args.target_dir}/{verilog_filename}\n") + return written_files + +def write_cc_filelist(filelist, out_filelist): + with open(out_filelist, "a") as df: + for path in filelist: + file = os.path.basename(path) + df.write(f"{args.target_dir}/{file}\n") def generate_copy(c, sfx): (cur_name, ext) = os.path.splitext(c) @@ -54,167 +112,96 @@ def generate_copy(c, sfx): bash(f"sed -i s/\"module {cur_name}\"/\"module {new_name}\"/ {new_file}") return new_file -def dfs_update_modules(tree, common_fnames, visited): - # List of direct submodules to update - childs_to_update = list() - for child in tree['instances']: - # We don't have to change stuff that are under the dut - if (child['module_name'] == args.dut): - continue - if dfs_update_modules(child, common_fnames, visited): - childs_to_update.append(child['module_name']) - if (child['module_name']) in common_fnames: - child['module_name'] = child['module_name'] + "_" + MODEL_SFX - - cur_module = tree['module_name'] - new_file = None - - # cur_file is in the common list, or is a ancestor of of them, generate a new file - if (cur_module in common_fnames) or len(childs_to_update) > 0: - new_file = 1 - - visited.add(cur_module) - return (new_file is not None) - -def bfs_update(tree, common_fnames, ext_dict, filelist): +def bfs_uniquify_modules(tree, common_fnames, verilog_module_filename): q = [(tree['instance_name'], tree['module_name'], tree['instances'], None)] - updated_submodule = set() + existing_modules = verilog_module_filename.keys() while len(q) != 0: front = q[0] q.pop(0) (inst, mod, child, parent) = front - try: - cur_file = mod + "." + ext_dict[mod][0] - except: - cur_file = mod + ".sv" + # external module + if mod not in existing_modules: + assert(len(child) == 0) + continue - mod_updated = False + cur_file = verilog_module_filename[mod] # if the module is common, make a copy & update its instance in its parent + new_mod = mod if mod in common_fnames: try: new_file = generate_copy(cur_file, MODEL_SFX) - filelist.append((mod, new_file)) if parent is not None and ((parent, mod) not in updated_submodule): - parent_file = os.path.join(args.gcpath, parent + "." + ext_dict[parent][0]) + parent_file = os.path.join(args.gcpath, verilog_module_filename[parent]) bash(f"sed -i s/\"{mod} \"/\"{mod}_{MODEL_SFX} \"/ {parent_file}") updated_submodule.add((parent, mod)) - mod_updated = True + + # add the uniquified module to the verilog_modul_filename dict + new_mod = mod + "_" + MODEL_SFX + verilog_module_filename[new_mod] = new_file except: print(f"No corresponding file for {cur_file}") - else: - filelist.append((mod, cur_file)) - - # set the parent module name - new_mod = mod - if mod_updated: - new_mod = mod + "_" + MODEL_SFX - ext_dict[new_mod] = ext_dict[mod] # traverse its children for c in child: if c['module_name'] != args.dut: q.append((c['instance_name'], c['module_name'], c['instances'], new_mod)) -def bfs_collect_modules(tree, child_to_ignore = None): - q = [(tree['instance_name'], tree['module_name'], tree['instances'])] +def dfs_update_modules(tree, common_fnames, visited): + # List of direct submodules to update + childs_to_update = list() + for child in tree['instances']: + # We don't have to change stuff that are under the dut + if (child['module_name'] == args.dut): + continue + if dfs_update_modules(child, common_fnames, visited): + childs_to_update.append(child['module_name']) + if (child['module_name']) in common_fnames: + child['module_name'] = child['module_name'] + "_" + MODEL_SFX - modules = list() - while len(q) != 0: - front = q[0] - q.pop(0) + cur_module = tree['module_name'] + new_file = None - (inst, mod, child) = front - modules.append(mod) - for c in child: - if c['module_name'] != child_to_ignore: - q.append((c['instance_name'], c['module_name'], c['instances'])) - return modules + # cur_file is in the common list, or is a ancestor of of them, generate a new file + if (cur_module in common_fnames) or len(childs_to_update) > 0: + new_file = 1 -def write_filelist(modules, out_file, files_written): - with open(out_file, "w") as df, \ - open(args.in_all_filelist) as fl: - # add paths that correspond to modules to output file - for path in fl: - writeOut = False - for dm in modules: - bm_ext = os.path.basename(path).split(".") - bm_ext.pop() - bm = ".".join(bm_ext) - print(dm, bm, bm_ext) - if (dm == bm) or (dm == bm_ext[0]): - writeOut = True - break - - # prepend the target directory to get filelist with absolute paths - if writeOut: - files_written.add(os.path.basename(path)) - if not args.target_dir in path: - df.write(f"{args.target_dir}/{path}") - else: - df.write(f"{path}") - -def write_filelist_model(modules, out_file, ext_dict, files_written): - with open(out_file, "w") as df: - for (m, fname) in modules: - if m in ext_dict.keys(): - if not args.target_dir in fname: - df.write(f"{args.target_dir}/{fname}\n") - else: - df.write(f"{fname}\n") - files_written.add(os.path.basename(fname)) - -def get_file_ext(all_filelist): - ext_dict = dict() - with open(all_filelist) as fl: - for path in fl: - fname = os.path.basename(path) - fname_strip = fname.strip().split(".") - ext = fname_strip[-1] - fname_strip.pop() - module = ".".join(fname_strip) - if module not in ext_dict.keys(): - ext_dict[module] = list() - ext_dict[module].append(ext) - return ext_dict + visited.add(cur_module) + return (new_file is not None) -def main(): +def uniquify_modules_under_model(modules_under_model, common_modules, verilog_module_filename): with open(args.model_hier_json) as imhj: imhj_data = json.load(imhj) - modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) - + visited = set() + bfs_uniquify_modules(imhj_data, common_modules, verilog_module_filename) + dfs_update_modules (imhj_data, common_modules, visited) - files_written = set() + with open(args.out_model_hier_json, "w+") as out_file: + json.dump(imhj_data, out_file, indent=2) - with open(args.top_hier_json) as imhj: - imhj_data = json.load(imhj) - modules_under_top = set(bfs_collect_modules(imhj_data)) +def main(): + verilog_module_filename = dict() + cc_filelist = list() + get_modules_in_filelist(verilog_module_filename, cc_filelist) - common_modules = modules_under_top.intersection(modules_under_model) - write_filelist(modules_under_top, args.out_dut_filelist, files_written) - ext_dict = get_file_ext(args.in_all_filelist) + modules_under_model = get_modules_under_hier(args.model_hier_json, args.dut) + modules_under_top = get_modules_under_hier(args.top_hier_json) + common_modules = modules_under_top.intersection(modules_under_model) - with open(args.model_hier_json) as imhj: - imhj_data = json.load(imhj) + # write top filelist + write_verilog_filelist(modules_under_top, verilog_module_filename, args.out_dut_filelist) - with open(args.out_model_hier_json, "w+") as out_file: - visited = set() - filelist = list() - bfs_update(imhj_data, common_modules, ext_dict, filelist) - dfs_update_modules(imhj_data, common_modules, visited) - json.dump(imhj_data, out_file, indent=2) - write_filelist_model(set(filelist), args.out_model_filelist, ext_dict, files_written) + # rename modules that are common + uniquify_modules_under_model(modules_under_model, common_modules, verilog_module_filename) + uniquified_modules_under_model = get_modules_under_hier(args.out_model_hier_json, args.dut) - with open(args.out_model_filelist, "a") as of, \ - open(args.in_all_filelist) as fl: - for path in fl: - fname = os.path.basename(path) - ext = fname.strip().split(".")[-1] - if (fname not in files_written) and (ext == "cc"): - of.write(f"{args.target_dir}/{fname}") + # write model filelist + write_verilog_filelist(uniquified_modules_under_model, verilog_module_filename, args.out_model_filelist) + write_cc_filelist (cc_filelist, args.out_model_filelist) + -if __name__ == "__main__": +if __name__=="__main__": main() From fec43fc14725859ec144c63f7c9a64f1c95cc512 Mon Sep 17 00:00:00 2001 From: joey0320 Date: Thu, 4 May 2023 21:51:00 -0700 Subject: [PATCH 10/10] corner case --- scripts/uniquify-module-names.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uniquify-module-names.py b/scripts/uniquify-module-names.py index 12a0843e54..75fc8c85db 100755 --- a/scripts/uniquify-module-names.py +++ b/scripts/uniquify-module-names.py @@ -53,7 +53,7 @@ def get_modules_in_verilog_file(file): for line in lines: words = line.split() if len(words) > 0 and words[0] == "module": - module_names.append(words[1].replace("(", "")) + module_names.append(words[1].replace("(", "").replace(")", "").replace(";", "")) return module_names def get_modules_in_filelist(verilog_module_filename, cc_filelist):