From ff3f634d033a9d7f23242260a3a6decfff2cea53 Mon Sep 17 00:00:00 2001 From: Heinrich Kuttler Date: Fri, 26 Nov 2021 19:55:09 +0100 Subject: [PATCH 1/5] Add glyph2tile array. This is to prepare a tile-based view. --- CMakeLists.txt | 7 +++++-- include/nleobs.h | 1 + nle/tests/test_nethack.py | 5 +++++ util/CMakeLists.txt | 12 ++++++++++++ win/rl/pynethack.cc | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 01cd70345..ba6db5b06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,7 +54,8 @@ add_compile_definitions( HACKDIR="${HACKDIR}" DEFAULT_WINDOW_SYS="rl" DLB - NOCWD_ASSUMPTIONS) + NOCWD_ASSUMPTIONS + NLE_USE_TILES) option(USE_SEEDING "Use seeding support in NLE" ON) @@ -70,6 +71,8 @@ set(NLE_INC ${nle_SOURCE_DIR}/include) set(NLE_DAT ${nle_SOURCE_DIR}/dat) set(NLE_UTIL ${nle_SOURCE_DIR}/util) set(NLE_DOC ${nle_SOURCE_DIR}/doc) +set(NLE_SSYS ${nle_SOURCE_DIR}/sys/share) +set(NLE_WIN ${nle_SOURCE_DIR}/win) set(NLE_SRC_GEN ${nle_BINARY_DIR}/src) set(NLE_INC_GEN ${nle_BINARY_DIR}/include) @@ -127,7 +130,7 @@ add_dependencies(rlmain util) # For pm.h. add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/pybind11) pybind11_add_module(_pynethack win/rl/pynethack.cc src/monst.c src/decl.c src/drawing.c src/objects.c) -target_link_libraries(_pynethack PUBLIC nethackdl) +target_link_libraries(_pynethack PUBLIC nethackdl tile) set_target_properties(_pynethack PROPERTIES CXX_STANDARD 14) target_include_directories(_pynethack PUBLIC ${NLE_INC_GEN}) add_dependencies(_pynethack util) # For pm.h. diff --git a/include/nleobs.h b/include/nleobs.h index eb2a6dda2..1e767ccf4 100644 --- a/include/nleobs.h +++ b/include/nleobs.h @@ -42,6 +42,7 @@ #define NLE_BL_CONDITION 25 /* condition bit mask */ /* #define NLE_ALLOW_SEEDING 1 */ /* Set in CMakeLists.txt if not disabled. */ +/* #define NLE_USE_TILES 1 */ /* Set in CMakeLists.txt. */ typedef struct nle_observation { int action; diff --git a/nle/tests/test_nethack.py b/nle/tests/test_nethack.py index f6c0f80c4..afab549f5 100644 --- a/nle/tests/test_nethack.py +++ b/nle/tests/test_nethack.py @@ -353,6 +353,11 @@ def test_symdef(self): assert darkroom_symdef.explanation == "dark part of a room" assert darkroom_symdef.color == 8 # NO_COLOR + def test_glyph2tile(self): + assert nethack.glyph2tile[nethack.GLYPH_MON_OFF] == 0 + assert nethack.glyph2tile[nethack.GLYPH_PET_OFF] == 0 + assert nethack.glyph2tile[nethack.GLYPH_DETECT_OFF] == 0 + class TestNethackGlanceObservation: @pytest.fixture diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt index dc9d0106a..39a630409 100644 --- a/util/CMakeLists.txt +++ b/util/CMakeLists.txt @@ -55,6 +55,11 @@ add_custom_command( OUTPUT ${NLE_INC_GEN}/pm.h COMMAND $ ARGS -p) +add_custom_command( + DEPENDS tilemap + OUTPUT ${NLE_SRC_GEN}/tile.c + COMMAND $) + add_custom_command( OUTPUT ${NLE_UTIL_GEN}/dgn_parser.c ${NLE_UTIL_GEN}/dgn_comp.h COMMAND bison -y --defines=${NLE_UTIL_GEN}/dgn_comp.h -o @@ -81,6 +86,13 @@ add_custom_command( add_custom_target(util DEPENDS ${MAKEDEFS_HEADERS} recover) +add_executable(tilemap ${NLE_WIN}/share/tilemap.c) +target_include_directories(tilemap PUBLIC ${NLE_INC} ${NLE_INC_GEN}) +add_dependencies(tilemap util) + +add_library(tile OBJECT ${NLE_SRC_GEN}/tile.c) +target_include_directories(tile PUBLIC ${NLE_INC} ${NLE_INC_GEN}) + # NOTE: util is dependent on these two add_dependencies(lev_comp util) add_dependencies(dgn_comp util) diff --git a/win/rl/pynethack.cc b/win/rl/pynethack.cc index 1fb598f2d..d40fa700e 100644 --- a/win/rl/pynethack.cc +++ b/win/rl/pynethack.cc @@ -25,6 +25,36 @@ extern "C" { #undef min #undef max +#ifdef NLE_USE_TILES +extern short glyph2tile[]; /* in tile.c (made from tilemap.c) */ + +/* Copy from dungeon.c. Necessary to add tile.c. + Can't add dungeon.c itself as it pulls in too much. */ + +/* are you in one of the Hell levels? */ +boolean +In_hell(d_level *lev) +{ + return (boolean) (dungeons[lev->dnum].flags.hellish); +} + +/* are you in the mines dungeon? */ +boolean +In_mines(d_level *lev) +{ + return (boolean) (lev->dnum == mines_dnum); +} + +/* are "lev1" and "lev2" actually the same? */ +boolean +on_level(d_level *lev1, d_level *lev2) +{ + return (boolean) (lev1->dnum == lev2->dnum + && lev1->dlevel == lev2->dlevel); +} +/* End of copy from dungeon.c */ +#endif + namespace py = pybind11; using namespace py::literals; @@ -501,6 +531,13 @@ PYBIND11_MODULE(_pynethack, m) mn.def("glyph_is_warning", [](int glyph) { return glyph_is_warning(glyph); }); +#ifdef NLE_USE_TILES + mn.attr("glyph2tile") = + py::memoryview::from_buffer(glyph2tile, /*shape=*/{ MAX_GLYPH }, + /*strides=*/{ sizeof(glyph2tile[0]) }, + /*readonly=*/true); +#endif + py::class_(mn, "permonst", "The permonst struct.") .def( "__init__", From 40af63cebb4b6e177cf286c21288094f4fd9abdb Mon Sep 17 00:00:00 2001 From: Heinrich Kuttler Date: Fri, 26 Nov 2021 20:09:52 +0100 Subject: [PATCH 2/5] For the GNU Makefile generator, creating NLE_SRC_GEN first is required. --- CMakeLists.txt | 12 +++++++++--- util/CMakeLists.txt | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ba6db5b06..17fb0db41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,9 +128,15 @@ add_dependencies(rlmain util) # For pm.h. # pybind11 python library. add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/pybind11) -pybind11_add_module(_pynethack win/rl/pynethack.cc src/monst.c src/decl.c - src/drawing.c src/objects.c) -target_link_libraries(_pynethack PUBLIC nethackdl tile) +pybind11_add_module( + _pynethack + win/rl/pynethack.cc + src/monst.c + src/decl.c + src/drawing.c + src/objects.c + $) +target_link_libraries(_pynethack PUBLIC nethackdl) set_target_properties(_pynethack PROPERTIES CXX_STANDARD 14) target_include_directories(_pynethack PUBLIC ${NLE_INC_GEN}) add_dependencies(_pynethack util) # For pm.h. diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt index 39a630409..8f0aa828e 100644 --- a/util/CMakeLists.txt +++ b/util/CMakeLists.txt @@ -22,7 +22,7 @@ set(MAKEDEFS_HEADERS ${NLE_INC_GEN}/date.h ${NLE_INC_GEN}/onames.h set(DLB_SRC dlb_main.c ${NLE_SRC}/dlb.c panic.c ${NLE_SRC}/alloc.c) -file(MAKE_DIRECTORY ${NLE_INC_GEN}) +file(MAKE_DIRECTORY ${NLE_INC_GEN} ${NLE_SRC_GEN}) add_executable(makedefs ${MAKEDEFS_SRC}) target_include_directories(makedefs PRIVATE ${NLE_INC} ${NLE_INC_GEN} From b1551309ce2a26e916d1aff5110b3170abc48a6e Mon Sep 17 00:00:00 2001 From: Heinrich Kuttler Date: Fri, 26 Nov 2021 20:44:01 +0100 Subject: [PATCH 3/5] Change clang-format workflow. This should fix it at clang-format version 12. --- .github/workflows/lint_cc.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint_cc.yml b/.github/workflows/lint_cc.yml index 37de29ccc..f0c43209d 100644 --- a/.github/workflows/lint_cc.yml +++ b/.github/workflows/lint_cc.yml @@ -21,7 +21,8 @@ jobs: name: "clang-format" runs-on: ubuntu-latest steps: - - name: Clone NLE repo - uses: actions/checkout@v2 - - name: Run clang - run: "./nle/scripts/run-clang-format -r win/rl src/nle.c sys/unix/nledl.c include/nle.h include/nledl.h include/nleobs.h" + - uses: actions/checkout@v2 + - uses: DoozyX/clang-format-lint-action@v0.12 + with: + source: 'win/rl src/nle.c sys/unix/nledl.c include/nle.h include/nledl.h include/nleobs.h' + clangFormatVersion: 12 From 78df47aba08572d06b7021c459f663b8011c7a4c Mon Sep 17 00:00:00 2001 From: Heinrich Kuttler Date: Fri, 26 Nov 2021 21:00:06 +0100 Subject: [PATCH 4/5] Include win/share in source distribution. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index a6bbaf25d..e058a7e49 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,6 +9,7 @@ recursive-include src * recursive-include sys * recursive-include util * recursive-include win/rl * +recursive-include win/share * recursive-include win/tty * recursive-include third_party * From e198bd526777524fbf748b3e3ddb80676106c085 Mon Sep 17 00:00:00 2001 From: Heinrich Kuttler Date: Fri, 26 Nov 2021 21:12:36 +0100 Subject: [PATCH 5/5] Remove run-clang-format script. --- .pre-commit-config.yaml | 4 +- Files.nle | 4 +- MANIFEST.in | 1 - nle/scripts/run-clang-format | 359 ----------------------------------- 4 files changed, 4 insertions(+), 364 deletions(-) delete mode 100755 nle/scripts/run-clang-format diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index feca6d46e..3c82927b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,9 +21,9 @@ repos: - id: clang-format name: clang-format description: Format files with clang-format - entry: nle/scripts/run-clang-format + entry: clang-format language: system - files: ^(src\/nle|include\/nle|win\/rl)\.(c|cc|cxx|cpp|cu|h|hpp|hxx|cuh|proto)$ + files: ^(src\/nle|include\/nle|win\/rl|sys\/unix\/nle).*\.(c|cc|cxx|cpp|cu|h|hpp|hxx|cuh|proto)$ - repo: https://github.com/pycqa/isort rev: 5.8.0 hooks: diff --git a/Files.nle b/Files.nle index 7f1e99211..59e1c1766 100644 --- a/Files.nle +++ b/Files.nle @@ -1,4 +1,4 @@ -Following Files.nh, this is a listing of all files that NLE adds to the NetHack +Following Files.nh, this is a listing of some files that NLE adds to the NetHack distribution. .: @@ -63,7 +63,7 @@ __init__.py actions.py nethack.py print_message.py ptyprocess.py nle/scripts: (files for the nle.scripts module) __init__.py check_nethack_speed.py collect_env.py lint_changed.sh -nh-clean-install play.py plot.py run-clang-format +nh-clean-install play.py plot.py test_raw_nethack.py ttyplay.py ttyrec.py nle/tests: diff --git a/MANIFEST.in b/MANIFEST.in index e058a7e49..753c4ea4e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -16,5 +16,4 @@ recursive-include third_party * recursive-exclude build * exclude nle/dashboard exclude nle/scripts/nh-clean-install.sh -exclude nle/scripts/run-clang-format.py exclude nle/scripts/line-changed.sh diff --git a/nle/scripts/run-clang-format b/nle/scripts/run-clang-format deleted file mode 100755 index b6a7317cf..000000000 --- a/nle/scripts/run-clang-format +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python3 -"""A wrapper script around clang-format, suitable for linting multiple files -and to use for continuous integration. - -This is an alternative API for the clang-format command line. -It runs over multiple files and directories in parallel. -A diff output is produced and a sensible exit code is returned. - -Taken originally from https://github.com/Sarcasm/run-clang-format -""" - -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -import codecs -import difflib -import fnmatch -import io -import multiprocessing -import os -import signal -import subprocess -import sys -import traceback -from functools import partial - -try: - from subprocess import DEVNULL # py3k -except ImportError: - DEVNULL = open(os.devnull, "wb") - - -DEFAULT_EXTENSIONS = "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" - - -class ExitStatus: - SUCCESS = 0 - DIFF = 1 - TROUBLE = 2 - - -def list_files(files, recursive=False, extensions=None, exclude=None): - if extensions is None: - extensions = [] - if exclude is None: - exclude = [] - - out = [] - for file in files: - if recursive and os.path.isdir(file): - for dirpath, dnames, fnames in os.walk(file): - fpaths = [os.path.join(dirpath, fname) for fname in fnames] - for pattern in exclude: - # os.walk() supports trimming down the dnames list - # by modifying it in-place, - # to avoid unnecessary directory listings. - dnames[:] = [ - x - for x in dnames - if not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) - ] - fpaths = [x for x in fpaths if not fnmatch.fnmatch(x, pattern)] - for f in fpaths: - ext = os.path.splitext(f)[1][1:] - if ext in extensions: - out.append(f) - else: - out.append(file) - return out - - -def make_diff(file, original, reformatted): - return list( - difflib.unified_diff( - original, - reformatted, - fromfile="{}\t(original)".format(file), - tofile="{}\t(reformatted)".format(file), - n=3, - ) - ) - - -class DiffError(Exception): - def __init__(self, message, errs=None): - super(DiffError, self).__init__(message) - self.errs = errs or [] - - -class UnexpectedError(Exception): - def __init__(self, message, exc=None): - super(UnexpectedError, self).__init__(message) - self.formatted_traceback = traceback.format_exc() - self.exc = exc - - -def run_clang_format_diff_wrapper(args, file): - try: - ret = run_clang_format_diff(args, file) - return ret - except DiffError: - raise - except Exception as e: - raise UnexpectedError("{}: {}: {}".format(file, e.__class__.__name__, e), e) - - -def run_clang_format_diff(args, file): - try: - with io.open(file, "r", encoding="utf-8") as f: - original = f.readlines() - except IOError as exc: - raise DiffError(str(exc)) - invocation = [args.clang_format_executable, file] - - # Use of utf-8 to decode the process output. - # - # Hopefully, this is the correct thing to do. - # - # It's done due to the following assumptions (which may be incorrect): - # - clang-format will returns the bytes read from the files as-is, - # without conversion, and it is already assumed that the files use utf-8. - # - if the diagnostics were internationalized, they would use utf-8: - # > Adding Translations to Clang - # > - # > Not possible yet! - # > Diagnostic strings should be written in UTF-8, - # > the client can translate to the relevant code page if needed. - # > Each translation completely replaces the format string - # > for the diagnostic. - # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation - # - # It's not pretty, due to Python 2 & 3 compatibility. - encoding_py3 = {} - if sys.version_info[0] >= 3: - encoding_py3["encoding"] = "utf-8" - - try: - proc = subprocess.Popen( - invocation, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - **encoding_py3, - ) - except OSError as exc: - raise DiffError( - "Command '{}' failed to start: {}".format( - subprocess.list2cmdline(invocation), exc - ) - ) - proc_stdout = proc.stdout - proc_stderr = proc.stderr - if sys.version_info[0] < 3: - # make the pipes compatible with Python 3, - # reading lines should output unicode - encoding = "utf-8" - proc_stdout = codecs.getreader(encoding)(proc_stdout) - proc_stderr = codecs.getreader(encoding)(proc_stderr) - # hopefully the stderr pipe won't get full and block the process - outs = list(proc_stdout.readlines()) - errs = list(proc_stderr.readlines()) - proc.wait() - if proc.returncode: - raise DiffError( - "Command '{}' returned non-zero exit status {}".format( - subprocess.list2cmdline(invocation), proc.returncode - ), - errs, - ) - return make_diff(file, original, outs), errs - - -def bold_red(s): - return "\x1b[1m\x1b[31m" + s + "\x1b[0m" - - -def colorize(diff_lines): - def bold(s): - return "\x1b[1m" + s + "\x1b[0m" - - def cyan(s): - return "\x1b[36m" + s + "\x1b[0m" - - def green(s): - return "\x1b[32m" + s + "\x1b[0m" - - def red(s): - return "\x1b[31m" + s + "\x1b[0m" - - for line in diff_lines: - if line[:4] in ["--- ", "+++ "]: - yield bold(line) - elif line.startswith("@@ "): - yield cyan(line) - elif line.startswith("+"): - yield green(line) - elif line.startswith("-"): - yield red(line) - else: - yield line - - -def print_diff(diff_lines, use_color): - if use_color: - diff_lines = colorize(diff_lines) - if sys.version_info[0] < 3: - sys.stdout.writelines((line.encode("utf-8") for line in diff_lines)) - else: - sys.stdout.writelines(diff_lines) - - -def print_trouble(prog, message, use_colors): - error_text = "error:" - if use_colors: - error_text = bold_red(error_text) - print("{}: {} {}".format(prog, error_text, message), file=sys.stderr) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--clang-format-executable", - metavar="EXECUTABLE", - help="path to the clang-format executable", - default="clang-format", - ) - parser.add_argument( - "--extensions", - help="comma separated list of file extensions (default: {})".format( - DEFAULT_EXTENSIONS - ), - default=DEFAULT_EXTENSIONS, - ) - parser.add_argument( - "-r", - "--recursive", - action="store_true", - help="run recursively over directories", - ) - parser.add_argument("files", metavar="file", nargs="+") - parser.add_argument("-q", "--quiet", action="store_true") - parser.add_argument( - "-j", - metavar="N", - type=int, - default=0, - help="run N clang-format jobs in parallel" " (default number of cpus + 1)", - ) - parser.add_argument( - "--color", - default="auto", - choices=["auto", "always", "never"], - help="show colored diff (default: auto)", - ) - parser.add_argument( - "-e", - "--exclude", - metavar="PATTERN", - action="append", - default=[], - help="exclude paths matching the given glob-like pattern(s)" - " from recursive search", - ) - - args = parser.parse_args() - - # use default signal handling, like diff return SIGINT value on ^C - # https://bugs.python.org/issue14229#msg156446 - signal.signal(signal.SIGINT, signal.SIG_DFL) - try: - signal.SIGPIPE - except AttributeError: - # compatibility, SIGPIPE does not exist on Windows - pass - else: - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - colored_stdout = False - colored_stderr = False - if args.color == "always": - colored_stdout = True - colored_stderr = True - elif args.color == "auto": - colored_stdout = sys.stdout.isatty() - colored_stderr = sys.stderr.isatty() - - version_invocation = [args.clang_format_executable, str("--version")] - try: - subprocess.check_call(version_invocation, stdout=DEVNULL) - except subprocess.CalledProcessError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - return ExitStatus.TROUBLE - except OSError as e: - print_trouble( - parser.prog, - "Command '{}' failed to start: {}".format( - subprocess.list2cmdline(version_invocation), e - ), - use_colors=colored_stderr, - ) - return ExitStatus.TROUBLE - - retcode = ExitStatus.SUCCESS - files = list_files( - args.files, - recursive=args.recursive, - exclude=args.exclude, - extensions=args.extensions.split(","), - ) - - if not files: - return - - njobs = args.j - if njobs == 0: - njobs = multiprocessing.cpu_count() + 1 - njobs = min(len(files), njobs) - - if njobs == 1: - # execute directly instead of in a pool, - # less overhead, simpler stacktraces - it = (run_clang_format_diff_wrapper(args, file) for file in files) - pool = None - else: - pool = multiprocessing.Pool(njobs) - it = pool.imap_unordered(partial(run_clang_format_diff_wrapper, args), files) - while True: - try: - outs, errs = next(it) - except StopIteration: - break - except DiffError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - retcode = ExitStatus.TROUBLE - sys.stderr.writelines(e.errs) - except UnexpectedError as e: - print_trouble(parser.prog, str(e), use_colors=colored_stderr) - sys.stderr.write(e.formatted_traceback) - retcode = ExitStatus.TROUBLE - # stop at the first unexpected error, - # something could be very wrong, - # don't process all files unnecessarily - if pool: - pool.terminate() - break - else: - sys.stderr.writelines(errs) - if outs == []: - continue - if not args.quiet: - print_diff(outs, use_color=colored_stdout) - if retcode == ExitStatus.SUCCESS: - retcode = ExitStatus.DIFF - return retcode - - -if __name__ == "__main__": - sys.exit(main())