From 97857087a20bc1cb95e03fef143cb74387d2f6c9 Mon Sep 17 00:00:00 2001 From: Jiakun Wang Date: Fri, 24 May 2024 15:03:50 -0400 Subject: [PATCH 1/7] feat: automated pip install, but manual environmental variable --- python/setup.py | 146 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 110 insertions(+), 36 deletions(-) diff --git a/python/setup.py b/python/setup.py index f96ecf0..c88d3ba 100644 --- a/python/setup.py +++ b/python/setup.py @@ -13,43 +13,114 @@ # limitations under the License. # import os +from pathlib import Path import sys import sysconfig -from setuptools import find_packages +from setuptools import find_packages, setup, Command +import subprocess + # need to use distutils.core for correct placement of cython dll -if "--inplace" in sys.argv: - from distutils.core import setup - from distutils.extension import Extension +if "--inplace" in sys.argv: + from distutils.core import setup + from distutils.extension import Extension else: - from setuptools import setup - from setuptools.extension import Extension + from setuptools import setup + from setuptools.extension import Extension + + + def config_cython(): - sys_cflags = sysconfig.get_config_var("CFLAGS") - try: - from Cython.Build import cythonize - ret = [] - path = "mirage/_cython" - for fn in os.listdir(path): - if not fn.endswith(".pyx"): - continue - ret.append(Extension( - "mirage.%s" % fn[:-4], - ["%s/%s" % (path, fn)], - include_dirs=["../include", "../deps/json/include", "../deps/cutlass/include", "/usr/local/cuda/include"], - libraries=["mirage_runtime", "cudadevrt", "cudart_static", "cudnn", "cublas", "cudart", "cuda", "z3"], - library_dirs=["../build", "../deps/z3/build", "/usr/local/cuda/lib64", "/usr/local/cuda/lib64/stubs"], - extra_compile_args=["-std=c++17"], - extra_link_args=["-fPIC"], - language="c++")) - return cythonize(ret, compiler_directives={"language_level" : 3}) - except ImportError: - print("WARNING: cython is not installed!!!") - return [] + sys_cflags = sysconfig.get_config_var("CFLAGS") + try: + from Cython.Build import cythonize + ret = [] + path = "mirage/_cython" + for fn in os.listdir(path): + if not fn.endswith(".pyx"): + continue + ret.append(Extension( + "mirage.%s" % fn[:-4], + ["%s/%s" % (path, fn)], + include_dirs=["../include", "../deps/json/include", "../deps/cutlass/include", "/usr/local/cuda/include"], + libraries=["mirage_runtime", "cudadevrt", "cudart_static", "cudnn", "cublas", "cudart", "cuda", "z3"], + library_dirs=["../build", "../deps/z3/build", "/usr/local/cuda/lib64", "/usr/local/cuda/lib64/stubs"], + extra_compile_args=["-std=c++17"], + extra_link_args=["-fPIC"], + language="c++")) + return cythonize(ret, compiler_directives={"language_level" : 3}) + except ImportError: + print("WARNING: cython is not installed!!!") + return [] + + + + +# install Z3 if not installed already +try: + subprocess.check_output(['z3', '--version']) + print('Z3 had already installed') +except FileNotFoundError: + print('Z3 not found, installing Z3 right now...') + try: + mirage_path = os.environ.get('MIRAGE') + if not mirage_path: + print("Please set the MIRAGE environment variable to the path of the mirage source directory.") + raise SystemExit(1) + z3_path = os.path.join(mirage_path, 'deps', 'z3') + build_dir = os.path.join(z3_path, 'build') + os.makedirs(build_dir, exist_ok=True) + print("finished making 'build' directory") + print(f"running cmake command at {build_dir}") + subprocess.check_call(['cmake', '..'], cwd=build_dir) + print("finished running cmake command") + print(f"running make command at {build_dir}") + subprocess.check_call(['make', '-j'], cwd=build_dir) + print("running make command") + + + + + # update LD_LIBRARY_PATH + os.environ['LD_LIBRARY_PATH'] = f"{build_dir}:{os.environ.get('LD_LIBRARY','')}" + print("Z3 installed successfully.") + + + + + except subprocess.CalledProcessError as e: + print("Failed to install Z3.") + raise SystemExit(e.returncode) + # build Mirage runtime library +try: + os.environ['CUDACXX'] = '/usr/local/cuda/bin/nvcc' + mirage_path = os.environ.get('MIRAGE') + if not mirage_path: + print("Please set the MIRAGE environment variable to the path of the mirage source directory.") + raise SystemExit(1) + z3_path = os.path.join(mirage_path, 'deps', 'z3') + build_dir = os.path.join(z3_path, 'build') + os.makedirs(build_dir, exist_ok=True) + subprocess.check_call(['cmake', '..'], cwd=build_dir) + subprocess.check_call(['make', '-j'], cwd=build_dir) + print("Mirage runtime library built successfully.") + + + + +except subprocess.CalledProcessError as e: + print("Failed to build runtime library.") + raise SystemExit(e.returncode) + + + setup_args = {} + + + #if not os.getenv('CONDA_BUILD'): # curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) # for i, path in enumerate(LIB_LIST): @@ -59,13 +130,16 @@ def config_cython(): # "data_files": [('mirage', LIB_LIST)] # } + + + setup(name='mirage', - version="0.1.0", - description="Mirage: A Multi-Level Superoptimizer for Tensor Algebra", - zip_safe=False, - install_requires=[], - packages=find_packages(), - url='https://github.com/mirage-project/mirage', - ext_modules=config_cython(), - #**setup_args, - ) + version="0.1.0", + description="Mirage: A Multi-Level Superoptimizer for Tensor Algebra", + zip_safe=False, + install_requires=[], + packages=find_packages(), + url='https://github.com/mirage-project/mirage', + ext_modules=config_cython(), + #**setup_args, + ) From 1456c16ef95a731993bf6b5ee07264cba3a5f690 Mon Sep 17 00:00:00 2001 From: Jiakun Wang Date: Tue, 28 May 2024 16:24:46 -0400 Subject: [PATCH 2/7] fix:small bug --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index c88d3ba..ce17417 100644 --- a/python/setup.py +++ b/python/setup.py @@ -99,7 +99,7 @@ def config_cython(): if not mirage_path: print("Please set the MIRAGE environment variable to the path of the mirage source directory.") raise SystemExit(1) - z3_path = os.path.join(mirage_path, 'deps', 'z3') + z3_path = os.path.join(mirage_path, 'deps', 'z3') build_dir = os.path.join(z3_path, 'build') os.makedirs(build_dir, exist_ok=True) subprocess.check_call(['cmake', '..'], cwd=build_dir) From b5792d54875bc2e9346e41931a2e7a3ede59d2d8 Mon Sep 17 00:00:00 2001 From: Jiakun Wang Date: Sat, 1 Jun 2024 23:50:40 -0400 Subject: [PATCH 3/7] feat: added requirements.txt, but encountered some other issues --- .vscode/sftp.json | 16 ++++++++++++++++ python/requirements.txt | 4 ++++ python/setup.py | 6 ++++-- 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 .vscode/sftp.json create mode 100644 python/requirements.txt diff --git a/.vscode/sftp.json b/.vscode/sftp.json new file mode 100644 index 0000000..41da1ea --- /dev/null +++ b/.vscode/sftp.json @@ -0,0 +1,16 @@ +{ + "name": "My Server", + "host": "catalyst-cluster.cs.cmu.edu", + "protocol": "sftp", + "port": 22, + "username": "jiakunw", + "remotePath": "/home/jiakunw/mirage", + "uploadOnSave": true, + "downloadOnOpen": true, + "ignore" : [ + ".vscode", + ".git", + ".DS_Store", + "admin" + ] +} \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000..ef1024f --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,4 @@ +cmake>=3.24 +cython>=0.28 +cuda>=11.0 +cudnn>=8.0 \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index ce17417..864a80a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -130,14 +130,16 @@ def config_cython(): # "data_files": [('mirage', LIB_LIST)] # } - +# Create requirements list from requirements.txt +with open(Path(__file__).parent / "requirements.txt", "r") as reqs_file: + requirements = reqs_file.read().strip().split("\n") setup(name='mirage', version="0.1.0", description="Mirage: A Multi-Level Superoptimizer for Tensor Algebra", zip_safe=False, - install_requires=[], + install_requires=requirements, packages=find_packages(), url='https://github.com/mirage-project/mirage', ext_modules=config_cython(), From 7e29ce415e5de47be3d7cba9e0077ff249a861b3 Mon Sep 17 00:00:00 2001 From: Jiakun Wang Date: Wed, 5 Jun 2024 01:26:11 -0400 Subject: [PATCH 4/7] fix: run_time lib configed correctly, but _cython does not exist --- python/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/setup.py b/python/setup.py index 864a80a..7aca91f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -71,6 +71,7 @@ def config_cython(): z3_path = os.path.join(mirage_path, 'deps', 'z3') build_dir = os.path.join(z3_path, 'build') os.makedirs(build_dir, exist_ok=True) + os.chdir(build_dir) print("finished making 'build' directory") print(f"running cmake command at {build_dir}") subprocess.check_call(['cmake', '..'], cwd=build_dir) @@ -101,7 +102,9 @@ def config_cython(): raise SystemExit(1) z3_path = os.path.join(mirage_path, 'deps', 'z3') build_dir = os.path.join(z3_path, 'build') + os.environ['Z3_DIR'] = build_dir os.makedirs(build_dir, exist_ok=True) + os.chdir(build_dir) subprocess.check_call(['cmake', '..'], cwd=build_dir) subprocess.check_call(['make', '-j'], cwd=build_dir) print("Mirage runtime library built successfully.") From 88864600e7640b5a9d1d9727070e98b84e3cf37c Mon Sep 17 00:00:00 2001 From: Jiakun Wang Date: Wed, 12 Jun 2024 21:16:40 -0400 Subject: [PATCH 5/7] fix: worked? --- python/requirements.txt | 4 +--- python/setup.py | 37 +++++++++++++++++++++++-------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index ef1024f..b167f4f 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,4 +1,2 @@ cmake>=3.24 -cython>=0.28 -cuda>=11.0 -cudnn>=8.0 \ No newline at end of file +cython>=0.28 \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index 6e82f88..c76964d 100644 --- a/python/setup.py +++ b/python/setup.py @@ -71,13 +71,18 @@ def config_cython(): try: mirage_path = os.environ.get('MIRAGE') if not mirage_path: - print("Please set the MIRAGE environment variable to the path of the mirage source directory.") - raise SystemExit(1) + print("Please set the MIRAGE environment variable to the path of the mirage source directory.") + raise SystemExit(1) + # ld_library_path = os.environ.get('LD_LIBRARY_PATH') + # if not ld_library_path: + # print("Please set the LD_LIBRARY_PATH environment variable.") + # raise SystemExit(1) + z3_path = os.path.join(mirage_path, 'deps', 'z3') build_dir = os.path.join(z3_path, 'build') os.makedirs(build_dir, exist_ok=True) os.chdir(build_dir) - print("finished making 'build' directory") + print(f"Changed to directory: {os.getcwd()}") print(f"running cmake command at {build_dir}") subprocess.check_call(['cmake', '..'], cwd=build_dir) print("finished running cmake command") @@ -85,11 +90,10 @@ def config_cython(): subprocess.check_call(['make', '-j'], cwd=build_dir) print("running make command") - - - # update LD_LIBRARY_PATH - os.environ['LD_LIBRARY_PATH'] = f"{build_dir}:{os.environ.get('LD_LIBRARY','')}" + print("here") + print(f"{build_dir}:{os.environ.get('LD_LIBRARY_PATH','')}") + os.environ['LD_LIBRARY_PATH'] = f"{build_dir}:{os.environ.get('LD_LIBRARY_PATH','LD_LIBRARY_PATH')}" print("Z3 installed successfully.") @@ -105,15 +109,20 @@ def config_cython(): if not mirage_path: print("Please set the MIRAGE environment variable to the path of the mirage source directory.") raise SystemExit(1) - z3_path = os.path.join(mirage_path, 'deps', 'z3') - build_dir = os.path.join(z3_path, 'build') - os.environ['Z3_DIR'] = build_dir + z3_path = os.path.join(mirage_path, 'deps', 'z3', 'build') + os.environ['Z3_DIR'] = z3_path + + os.makedirs(mirage_path, exist_ok=True) + os.chdir(mirage_path) + build_dir = os.path.join(mirage_path, 'build') + + # Create the build directory if it does not exist os.makedirs(build_dir, exist_ok=True) - os.chdir(build_dir) - subprocess.check_call(['cmake', '..'], cwd=build_dir) - subprocess.check_call(['make', '-j'], cwd=build_dir) + + subprocess.check_call(['cmake', '..'], cwd=build_dir, env=os.environ.copy()) + subprocess.check_call(['make', '-j'], cwd=build_dir, env=os.environ.copy()) print("Mirage runtime library built successfully.") - + # import pdb; pdb.set_trace() From d10f439f9acfa7d4bcfbaa1d01a97ead666bdc9a Mon Sep 17 00:00:00 2001 From: Jiakun Wang Date: Tue, 18 Jun 2024 19:56:16 -0400 Subject: [PATCH 6/7] fix: no .vscode --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index c8f7baa..7bf8615 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ build/ bin/ /.cache/ + +#vscode +.vscode/ \ No newline at end of file From 73e85122f0e15dc430e0fb636037360b0b5fe4fe Mon Sep 17 00:00:00 2001 From: jiakunw <89547922+jiakunw@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:04:19 -0400 Subject: [PATCH 7/7] fix: "re-Deleted .vscode directory" --- .vscode/sftp.json | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .vscode/sftp.json diff --git a/.vscode/sftp.json b/.vscode/sftp.json deleted file mode 100644 index 41da1ea..0000000 --- a/.vscode/sftp.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "My Server", - "host": "catalyst-cluster.cs.cmu.edu", - "protocol": "sftp", - "port": 22, - "username": "jiakunw", - "remotePath": "/home/jiakunw/mirage", - "uploadOnSave": true, - "downloadOnOpen": true, - "ignore" : [ - ".vscode", - ".git", - ".DS_Store", - "admin" - ] -} \ No newline at end of file