From 7b1e629b611a9c8541d210bdd167e53752aecb42 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Thu, 13 May 2021 07:31:09 -0700 Subject: [PATCH] Use python3's pathlib in test code. NFC There are plans underway to start using pathlib more in emscripten. See: #14074 This change a designed to test the water by using `pathlib` in test code. This allows us to use unix-like paths throughout the tests that will get converted to windows-paths on windows machines automatically by the pathlib library. --- tests/jsrun.py | 2 +- tests/runner.py | 31 +-- tests/test_browser.py | 455 ++++++++++++++++++------------------ tests/test_core.py | 499 ++++++++++++++++++++-------------------- tests/test_other.py | 327 +++++++++++++------------- tests/test_posixtest.py | 2 +- tests/test_sanity.py | 5 +- tools/shared.py | 1 + 8 files changed, 665 insertions(+), 657 deletions(-) diff --git a/tests/jsrun.py b/tests/jsrun.py index 6e5a16522f98c..196add5b2b722 100644 --- a/tests/jsrun.py +++ b/tests/jsrun.py @@ -62,7 +62,7 @@ def check_engine(engine): if engine_path not in WORKING_ENGINES: logging.debug('Checking JS engine %s' % engine) try: - output = run_js(shared.path_from_root('tests', 'hello_world.js'), engine, skip_check=True) + output = run_js(shared.path_from_root('tests/hello_world.js'), engine, skip_check=True) if 'hello, world!' in output: WORKING_ENGINES[engine_path] = True else: diff --git a/tests/runner.py b/tests/runner.py index 105a88622adea..259f4871d5b22 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -43,6 +43,7 @@ import time import unittest import webbrowser +from pathlib import Path from http.server import HTTPServer, SimpleHTTPRequestHandler from urllib.parse import unquote, unquote_plus @@ -65,7 +66,7 @@ def path_from_root(*pathelems): """Construct a path relative to the emscripten root directory.""" - return os.path.join(__rootpath__, *pathelems) + return os.fspath(Path(__rootpath__, *pathelems)) sys.path.append(path_from_root('third_party/websockify')) @@ -111,7 +112,7 @@ def delete_contents(pathname): def test_file(*path_components): """Construct a path relative to the emscripten "tests" directory.""" - return os.path.join(TEST_ROOT, *path_components) + return os.fspath(Path(TEST_ROOT, *path_components)) # checks if browser testing is enabled @@ -241,8 +242,8 @@ def modified(self): def ensure_dir(dirname): - if not os.path.isdir(dirname): - os.makedirs(dirname) + dirname = Path(dirname) + dirname.mkdir(parents=True, exist_ok=True) def limit_size(string, maxbytes=800000 * 20, maxlines=100000, max_line=5000): @@ -259,14 +260,16 @@ def limit_size(string, maxbytes=800000 * 20, maxlines=100000, max_line=5000): def create_file(name, contents, binary=False): - assert not os.path.isabs(name) - mode = 'wb' if binary else 'w' - with open(name, mode) as f: - f.write(contents) + name = Path(name) + assert not name.is_absolute() + if binary: + name.write_bytes(contents) + else: + name.write_text(contents) def make_executable(name): - os.chmod(name, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) + Path(name).chmod(stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) # The core test modes @@ -1100,8 +1103,8 @@ def get_poppler_library(self, env_init=None): self.set_setting('ERROR_ON_UNDEFINED_SYMBOLS', 0) self.emcc_args += [ - '-I' + test_file('third_party', 'freetype', 'include'), - '-I' + test_file('third_party', 'poppler', 'include') + '-I' + test_file('third_party/freetype/include'), + '-I' + test_file('third_party/poppler/include') ] freetype = self.get_freetype_library() @@ -1613,10 +1616,12 @@ def build_library(name, generated_libs = [generated_libs] source_dir = test_file(name.replace('_native', '')) - project_dir = os.path.join(build_dir, name) + project_dir = Path(build_dir, name) + print(project_dir) if os.path.exists(project_dir): shutil.rmtree(project_dir) - shutil.copytree(source_dir, project_dir) # Useful in debugging sometimes to comment this out, and two lines above + # Useful in debugging sometimes to comment this out, and two lines above + shutil.copytree(source_dir, project_dir) generated_libs = [os.path.join(project_dir, lib) for lib in generated_libs] if native: diff --git a/tests/test_browser.py b/tests/test_browser.py index fa48ae734fd25..48b8a926b3ea1 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -17,6 +17,7 @@ import webbrowser import zlib from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path from urllib.request import urlopen from runner import BrowserCore, RunnerCore, path_from_root, has_browser, EMTEST_BROWSER, Reporting @@ -216,18 +217,15 @@ def test_zzz_html_source_map(self): ''') def test_emscripten_log(self): - self.btest_exit(test_file('emscripten_log', 'emscripten_log.cpp'), + self.btest_exit(test_file('emscripten_log/emscripten_log.cpp'), args=['--pre-js', path_from_root('src', 'emscripten-source-map.min.js'), '-gsource-map']) def test_preload_file(self): - absolute_src_path = os.path.join(self.get_dir(), 'somefile.txt').replace('\\', '/') - open(absolute_src_path, 'w').write('''load me right before running the code please''') + create_file('somefile.txt', 'load me right before running the code please') + create_file('.somefile.txt', 'load me right before running the code please') + create_file('some@file.txt', 'load me right before running the code please') - absolute_src_path2 = os.path.join(self.get_dir(), '.somefile.txt').replace('\\', '/') - open(absolute_src_path2, 'w').write('''load me right before running the code please''') - - absolute_src_path3 = os.path.join(self.get_dir(), 'some@file.txt').replace('\\', '/') - open(absolute_src_path3, 'w').write('''load me right before running the code please''') + absolute_src_path = os.path.abspath('somefile.txt') def make_main(path): print('make main at', path) @@ -281,7 +279,7 @@ def make_main(path): else: # All 7-bit non-alphanumeric non-control code ASCII characters except /, : and \ are allowed. tricky_filename = '!#$%&\'()+,-. ;=@[]^_`{}~ "*<>?|.txt' - open(os.path.join(self.get_dir(), tricky_filename), 'w').write('''load me right before running the code please''') + create_file(tricky_filename, 'load me right before running the code please') make_main(tricky_filename) # As an Emscripten-specific feature, the character '@' must be escaped in the form '@@' to not confuse with the 'src@dst' notation. self.compile_btest(['main.cpp', '--preload-file', tricky_filename.replace('@', '@@'), '-o', 'page.html']) @@ -377,12 +375,13 @@ def test_preload_file_with_manual_data_download(self): shutil.copyfile(test_file('manual_download_data.html'), 'manual_download_data.html') self.run_browser('manual_download_data.html', 'Hello!', '/report_result?1') - # Tests that if the output files have single or double quotes in them, that it will be handled by correctly escaping the names. + # Tests that if the output files have single or double quotes in them, that it will be handled by + # correctly escaping the names. def test_output_file_escaping(self): tricky_part = '\'' if WINDOWS else '\' and \"' # On Windows, files/directories may not contain a double quote character. On non-Windowses they can, so test that. d = 'dir with ' + tricky_part - abs_d = os.path.join(self.get_dir(), d) + abs_d = os.path.abspath(d) ensure_dir(abs_d) txt = 'file with ' + tricky_part + '.txt' abs_txt = os.path.join(abs_d, txt) @@ -410,7 +409,7 @@ def test_output_file_escaping(self): data_js_file = os.path.join(abs_d, 'file with ' + tricky_part + '.js') self.run_process([FILE_PACKAGER, data_file, '--use-preload-cache', '--indexedDB-name=testdb', '--preload', abs_txt + '@' + txt, '--js-output=' + data_js_file]) page_file = os.path.join(d, 'file with ' + tricky_part + '.html') - abs_page_file = os.path.join(self.get_dir(), page_file) + abs_page_file = os.path.abspath(page_file) self.compile_btest([cpp, '--pre-js', data_js_file, '-o', abs_page_file, '-s', 'FORCE_FILESYSTEM']) self.run_browser(page_file, '|load me right before|.', '/report_result?0') @@ -525,9 +524,9 @@ def make_main(path): def test_multifile(self): # a few files inside a directory - ensure_dir(os.path.join('subdirr', 'moar')) - create_file(os.path.join('subdirr', 'data1.txt'), '1214141516171819') - create_file(os.path.join('subdirr', 'moar', 'data2.txt'), '3.14159265358979') + ensure_dir('subdirr/moar') + create_file('subdirr/data1.txt', '1214141516171819') + create_file('subdirr/moar/data2.txt', '3.14159265358979') create_file('main.cpp', r''' #include #include @@ -568,7 +567,7 @@ def test_custom_file_package_url(self): # a few files inside a directory ensure_dir('subdirr') ensure_dir('cdn') - create_file(os.path.join('subdirr', 'data1.txt'), '1214141516171819') + create_file(Path('subdirr/data1.txt'), '1214141516171819') # change the file package base dir to look in a "cdn". note that normally # you would add this in your own custom html file etc., and not by # modifying the existing shell in this manner @@ -593,7 +592,7 @@ def test_custom_file_package_url(self): ''') self.compile_btest(['main.cpp', '--shell-file', 'shell.html', '--preload-file', 'subdirr/data1.txt', '-o', 'test.html']) - shutil.move('test.data', os.path.join('cdn', 'test.data')) + shutil.move('test.data', Path('cdn/test.data')) self.run_browser('test.html', '', '/report_result?1') def test_missing_data_throws_error(self): @@ -660,7 +659,7 @@ def test(): # test() def test_dev_random(self): - self.btest(os.path.join('filesystem', 'dev_random.cpp'), expected='0') + self.btest(Path('filesystem/dev_random.cpp'), expected='0') def test_sdl_swsurface(self): self.btest('sdl_swsurface.c', args=['-lSDL', '-lGL'], expected='1') @@ -1216,7 +1215,7 @@ def test_webgl_context_attributes(self): # Copy common code file to temporary directory filepath = test_file('test_webgl_context_attributes_common.c') - temp_filepath = os.path.join(self.get_dir(), os.path.basename(filepath)) + temp_filepath = os.path.basename(filepath) shutil.copyfile(filepath, temp_filepath) # perform tests with attributes activated @@ -1280,13 +1279,13 @@ def test_file_db(self): def test_fs_idbfs_sync(self): for extra in [[], ['-DEXTRA_WORK']]: secret = str(time.time()) - self.btest(test_file('fs', 'test_idbfs_sync.c'), '1', args=['-lidbfs.js', '-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_test,_success', '-lidbfs.js']) - self.btest(test_file('fs', 'test_idbfs_sync.c'), '1', args=['-lidbfs.js', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_test,_success', '-lidbfs.js'] + extra) + self.btest(test_file('fs/test_idbfs_sync.c'), '1', args=['-lidbfs.js', '-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_test,_success', '-lidbfs.js']) + self.btest(test_file('fs/test_idbfs_sync.c'), '1', args=['-lidbfs.js', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_test,_success', '-lidbfs.js'] + extra) def test_fs_idbfs_sync_force_exit(self): secret = str(time.time()) - self.btest(test_file('fs', 'test_idbfs_sync.c'), '1', args=['-lidbfs.js', '-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_test,_success', '-s', 'EXIT_RUNTIME', '-DFORCE_EXIT', '-lidbfs.js']) - self.btest(test_file('fs', 'test_idbfs_sync.c'), '1', args=['-lidbfs.js', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_test,_success', '-s', 'EXIT_RUNTIME', '-DFORCE_EXIT', '-lidbfs.js']) + self.btest(test_file('fs/test_idbfs_sync.c'), '1', args=['-lidbfs.js', '-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_test,_success', '-s', 'EXIT_RUNTIME', '-DFORCE_EXIT', '-lidbfs.js']) + self.btest(test_file('fs/test_idbfs_sync.c'), '1', args=['-lidbfs.js', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_test,_success', '-s', 'EXIT_RUNTIME', '-DFORCE_EXIT', '-lidbfs.js']) def test_fs_idbfs_fsync(self): # sync from persisted state into memory before main() @@ -1305,13 +1304,13 @@ def test_fs_idbfs_fsync(self): args = ['--pre-js', 'pre.js', '-lidbfs.js', '-s', 'EXIT_RUNTIME', '-s', 'ASYNCIFY'] secret = str(time.time()) - self.btest(test_file('fs', 'test_idbfs_fsync.c'), '1', args=args + ['-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_success', '-lidbfs.js']) - self.btest(test_file('fs', 'test_idbfs_fsync.c'), '1', args=args + ['-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_success', '-lidbfs.js']) + self.btest(test_file('fs/test_idbfs_fsync.c'), '1', args=args + ['-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_success', '-lidbfs.js']) + self.btest(test_file('fs/test_idbfs_fsync.c'), '1', args=args + ['-DSECRET=\"' + secret + '\"', '-s', 'EXPORTED_FUNCTIONS=_main,_success', '-lidbfs.js']) def test_fs_memfs_fsync(self): args = ['-s', 'ASYNCIFY', '-s', 'EXIT_RUNTIME'] secret = str(time.time()) - self.btest(test_file('fs', 'test_memfs_fsync.c'), '1', args=args + ['-DSECRET=\"' + secret + '\"']) + self.btest(test_file('fs/test_memfs_fsync.c'), '1', args=args + ['-DSECRET=\"' + secret + '\"']) def test_fs_workerfs_read(self): secret = 'a' * 10 @@ -1328,41 +1327,41 @@ def test_fs_workerfs_read(self): }, '/work'); }; ''' % (secret, secret2)) - self.btest(test_file('fs', 'test_workerfs_read.c'), '1', args=['-lworkerfs.js', '--pre-js', 'pre.js', '-DSECRET=\"' + secret + '\"', '-DSECRET2=\"' + secret2 + '\"', '--proxy-to-worker', '-lworkerfs.js']) + self.btest(test_file('fs/test_workerfs_read.c'), '1', args=['-lworkerfs.js', '--pre-js', 'pre.js', '-DSECRET=\"' + secret + '\"', '-DSECRET2=\"' + secret2 + '\"', '--proxy-to-worker', '-lworkerfs.js']) def test_fs_workerfs_package(self): create_file('file1.txt', 'first') ensure_dir('sub') - open(os.path.join('sub', 'file2.txt'), 'w').write('second') - self.run_process([FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', os.path.join('sub', 'file2.txt'), '--separate-metadata', '--js-output=files.js']) - self.btest(os.path.join('fs', 'test_workerfs_package.cpp'), '1', args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js']) + open(Path('sub/file2.txt'), 'w').write('second') + self.run_process([FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', Path('sub/file2.txt'), '--separate-metadata', '--js-output=files.js']) + self.btest(Path('fs/test_workerfs_package.cpp'), '1', args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js']) def test_fs_lz4fs_package(self): # generate data ensure_dir('subdir') create_file('file1.txt', '0123456789' * (1024 * 128)) - open(os.path.join('subdir', 'file2.txt'), 'w').write('1234567890' * (1024 * 128)) + open(Path('subdir/file2.txt'), 'w').write('1234567890' * (1024 * 128)) random_data = bytearray(random.randint(0, 255) for x in range(1024 * 128 * 10 + 1)) random_data[17] = ord('X') open('file3.txt', 'wb').write(random_data) # compress in emcc, -s LZ4=1 tells it to tell the file packager print('emcc-normal') - self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['-s', 'LZ4=1', '--preload-file', 'file1.txt', '--preload-file', 'subdir/file2.txt', '--preload-file', 'file3.txt']) - assert os.path.getsize('file1.txt') + os.path.getsize(os.path.join('subdir', 'file2.txt')) + os.path.getsize('file3.txt') == 3 * 1024 * 128 * 10 + 1 + self.btest(Path('fs/test_lz4fs.cpp'), '2', args=['-s', 'LZ4=1', '--preload-file', 'file1.txt', '--preload-file', 'subdir/file2.txt', '--preload-file', 'file3.txt']) + assert os.path.getsize('file1.txt') + os.path.getsize(Path('subdir/file2.txt')) + os.path.getsize('file3.txt') == 3 * 1024 * 128 * 10 + 1 assert os.path.getsize('test.data') < (3 * 1024 * 128 * 10) / 2 # over half is gone print(' emcc-opts') - self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['-s', 'LZ4=1', '--preload-file', 'file1.txt', '--preload-file', 'subdir/file2.txt', '--preload-file', 'file3.txt', '-O2']) + self.btest(Path('fs/test_lz4fs.cpp'), '2', args=['-s', 'LZ4=1', '--preload-file', 'file1.txt', '--preload-file', 'subdir/file2.txt', '--preload-file', 'file3.txt', '-O2']) # compress in the file packager, on the server. the client receives compressed data and can just use it. this is typical usage print('normal') out = subprocess.check_output([FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', 'subdir/file2.txt', 'file3.txt', '--lz4']) open('files.js', 'wb').write(out) - self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['--pre-js', 'files.js', '-s', 'LZ4=1', '-s', 'FORCE_FILESYSTEM']) + self.btest(Path('fs/test_lz4fs.cpp'), '2', args=['--pre-js', 'files.js', '-s', 'LZ4=1', '-s', 'FORCE_FILESYSTEM']) print(' opts') - self.btest(os.path.join('fs', 'test_lz4fs.cpp'), '2', args=['--pre-js', 'files.js', '-s', 'LZ4=1', '-s', 'FORCE_FILESYSTEM', '-O2']) + self.btest(Path('fs/test_lz4fs.cpp'), '2', args=['--pre-js', 'files.js', '-s', 'LZ4=1', '-s', 'FORCE_FILESYSTEM', '-O2']) print(' modularize') - self.compile_btest([test_file('fs', 'test_lz4fs.cpp'), '--pre-js', 'files.js', '-s', 'LZ4=1', '-s', 'FORCE_FILESYSTEM', '-s', 'MODULARIZE=1']) + self.compile_btest([test_file('fs/test_lz4fs.cpp'), '--pre-js', 'files.js', '-s', 'LZ4=1', '-s', 'FORCE_FILESYSTEM', '-s', 'MODULARIZE=1']) create_file('a.html', '''