Skip to content
This repository has been archived by the owner on May 6, 2024. It is now read-only.

Commit

Permalink
Add ttyrec_version to the database schema.
Browse files Browse the repository at this point in the history 
This allows the dataset to know which version of ttyrec is being read,
which will allow the downstream Dataset object to chose the right way to read
the arrays in future.

This ttyrec version is always 1 for 'altorg' or old style ttyrecs, and
currently 2 for any nle generated ttyrecs.
  • Loading branch information
@cdmatters
cdmatters committed May 9, 2022
1 parent 06f016b commit 037d554
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 21 deletions.
18 changes: 14 additions & 4 deletions nle/dataset/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,13 @@ def set_root(dataset_name, root):
conn.commit()


def get_ttyrec_version(dataset_name, conn=None):
with db(conn) as conn:
return conn.execute(
"SELECT ttyrec_version FROM roots WHERE dataset_name=?", (dataset_name,)
).fetchone()[0]


def get_most_recent_games(n=1, conn=None):
with db(conn=conn) as conn:
c = conn.execute("SELECT gameid FROM games ORDER BY gameid DESC LIMIT ?", (n,))
Expand Down Expand Up @@ -145,9 +152,11 @@ def delete_games_with_select(select, not_in=False, conn=None, commit=True):
conn.commit()


def create_dataset(dataset_name, root, conn=None, commit=True):
def create_dataset(dataset_name, root, ttyrec_version=0, conn=None, commit=True):
with db(conn, rw=True) as conn:
conn.execute("INSERT INTO roots VALUES (?, ?)", (dataset_name, root))
conn.execute(
"INSERT INTO roots VALUES (?, ?, ?)", (dataset_name, root, ttyrec_version)
)
conn.execute("UPDATE meta SET mtime = ?", (time.time(),))
if commit:
conn.commit()
Expand Down Expand Up @@ -237,8 +246,9 @@ def create(filename=DB):
c.execute(
"""CREATE TABLE roots
(
dataset_name TEXT PRIMARY KEY,
root TEXT
dataset_name TEXT PRIMARY KEY,
root TEXT,
ttyrec_version INTEGER
)"""
)

Expand Down
10 changes: 8 additions & 2 deletions nle/dataset/populate_db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ def add_altorg_directory(path, name, filename=db.DB):
c = conn.cursor()

# 1. Check if the dataset name exists, and add the root.
db.create_dataset(name, root, conn=c, commit=False)
# NB: alt.org ttyrecs are version 1, and have suffix "ttyrec.bz2"
db.create_dataset(name, root, ttyrec_version=1, conn=c, commit=False)

# 2. Add games from xlogfile to `games` table, then `datasets` table.
for xlogfile in reversed(
Expand Down Expand Up @@ -267,7 +268,12 @@ def add_nledata_directory(path, name, filename=db.DB):
resets = set(int(i.split(".")[-3]) for i in files)

assert len(versions) == 1, "Cannot add ttyrecs with different versions"
version = str(versions.pop())
version = versions.pop()
c.execute(
"UPDATE roots SET ttyrec_version = ? WHERE dataset_name = ?",
(version, name),
)
version = str(version)

def filter(gen):
# The `xlogfile` may have more rows than files in directory
Expand Down
24 changes: 12 additions & 12 deletions nle/tests/altorg/db.json
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
[
["FlamingKetchup/2019-07-25.22:03:29.ttyrec.bz2", 0, 22, 1649954022.624912, 8, 8, "3.6.3", 1254, 2, 4, 5, 0, 43, 1, 20190726, 20190725, 5, "Kni", "Hum", "Mal", "Law", "FlamingKetchup", "killed by an elf zombie while frozen by a monster's gaze", "0xf88", 2662, "0x0", 7243, 1564092226, 1564100187, "Mal", "Law", "0x4", 8, "altorgtest", "altorgtest", "/path/to/altorg"],
["FlamingKetchup/2019-07-25.22:08:03.ttyrec.bz2", 1, 22, 1649954022.628631, 8, 8, "3.6.3", 1254, 2, 4, 5, 0, 43, 1, 20190726, 20190725, 5, "Kni", "Hum", "Mal", "Law", "FlamingKetchup", "killed by an elf zombie while frozen by a monster's gaze", "0xf88", 2662, "0x0", 7243, 1564092226, 1564100187, "Mal", "Law", "0x4", 8, "altorgtest", "altorgtest", "/path/to/altorg"],
["FlamingKetchup/2019-07-25.22:16:19.ttyrec.bz2", 2, 22, 1649954022.63341, 8, 8, "3.6.3", 1254, 2, 4, 5, 0, 43, 1, 20190726, 20190725, 5, "Kni", "Hum", "Mal", "Law", "FlamingKetchup", "killed by an elf zombie while frozen by a monster's gaze", "0xf88", 2662, "0x0", 7243, 1564092226, 1564100187, "Mal", "Law", "0x4", 8, "altorgtest", "altorgtest", "/path/to/altorg"],
["aanderson/2019-07-21.15:10:27.ttyrec.bz2", 0, 22, 1649954022.644775, 9, 9, "3.6.3", 533, 0, 4, 4, 0, 20, 1, 20190729, 20190721, 5, "Wiz", "Elf", "Mal", "Cha", "aanderson", "killed by a wererat", "0xf88", 882, "0x0", 7995, 1563721850, 1564435194, "Mal", "Cha", "0x4", 9, "altorgtest", "altorgtest", "/path/to/altorg"],
["aanderson/2019-07-29.21:13:49.ttyrec.bz2", 1, 22, 1649954022.649788, 9, 9, "3.6.3", 533, 0, 4, 4, 0, 20, 1, 20190729, 20190721, 5, "Wiz", "Elf", "Mal", "Cha", "aanderson", "killed by a wererat", "0xf88", 882, "0x0", 7995, 1563721850, 1564435194, "Mal", "Cha", "0x4", 9, "altorgtest", "altorgtest", "/path/to/altorg"],
["FlamingKetchup/2019-07-30.16:06:23.ttyrec.bz2", 0, 22, 1649954022.640018, 10, 10, "3.6.3", 746, 2, 5, 6, 0, 43, 1, 20190730, 20190730, 5, "Val", "Hum", "Fem", "Law", "FlamingKetchup", "killed by a gnome", "0xfc0", 1875, "0x0", 1577, 1564502799, 1564504378, "Fem", "Law", "0x4", 10, "altorgtest", "altorgtest", "/path/to/altorg"],
["aanderson/2019-08-03.16:56:46.ttyrec.bz2", 0, 22, 1649954022.655513, 11, 11, "3.6.3", 464, 0, 3, 5, 0, 25, 1, 20190803, 20190803, 5, "Wiz", "Gno", "Mal", "Neu", "aanderson", "killed by a homunculus while sleeping", "0xf88", 1140, "0x0", 764, 1564851435, 1564852201, "Mal", "Neu", "0x4", 11, "altorgtest", "altorgtest", "/path/to/altorg"],
["aanderson/2019-08-04.09:40:55.ttyrec.bz2", 0, 22, 1649954022.673842, 13, 13, "3.6.3", 1035, 2, 6, 6, 0, 46, 1, 20190804, 20190804, 5, "Wiz", "Gno", "Mal", "Neu", "aanderson", "killed by a rothe", "0xe88", 1003, "0x0", 8447, 1564911667, 1564935411, "Mal", "Neu", "0x4", 13, "altorgtest", "altorgtest", "/path/to/altorg"],
["aanderson/2019-08-04.15:56:08.ttyrec.bz2", 1, 22, 1649954022.678656, 13, 13, "3.6.3", 1035, 2, 6, 6, 0, 46, 1, 20190804, 20190804, 5, "Wiz", "Gno", "Mal", "Neu", "aanderson", "killed by a rothe", "0xe88", 1003, "0x0", 8447, 1564911667, 1564935411, "Mal", "Neu", "0x4", 13, "altorgtest", "altorgtest", "/path/to/altorg"],
["aartist/2008-09-08.06:45:16.ttyrec.bz2", 0, 22, 1649954022.692907, 14, 14, "3.4.3", 537, 0, 2, 2, 31, 31, 0, 20080909, 20080908, 5, "Val", "Hum", "Fem", "Neu", "aartist", "quit", "0xfce", 1346, "0x0", 1149, 1220856342, 1220926886, "Fem", "Neu", "-1", 14, "altorgtest", "altorgtest", "/path/to/altorg"],
["aartist/2008-09-09.02:02:56.ttyrec.bz2", 1, 22, 1649954022.699497, 14, 14, "3.4.3", 537, 0, 2, 2, 31, 31, 0, 20080909, 20080908, 5, "Val", "Hum", "Fem", "Neu", "aartist", "quit", "0xfce", 1346, "0x0", 1149, 1220856342, 1220926886, "Fem", "Neu", "-1", 14, "altorgtest", "altorgtest", "/path/to/altorg"],
["aartist/2008-09-09.02:21:28.ttyrec.bz2", 0, 22, 1649954022.704478, 15, 15, "3.4.3", 134, 0, 2, 2, 14, 15, 1, 20080909, 20080909, 5, "Sam", "Hum", "Mal", "Law", "aartist", "choked on a gunyoki", "0xf88", 251, "0x0", 373, 1220926913, 1220927313, "Mal", "Law", "-1", 15, "altorgtest", "altorgtest", "/path/to/altorg"]
["FlamingKetchup/2019-07-25.22:03:29.ttyrec.bz2", 0, 22, 1649954022.624912, 8, 8, "3.6.3", 1254, 2, 4, 5, 0, 43, 1, 20190726, 20190725, 5, "Kni", "Hum", "Mal", "Law", "FlamingKetchup", "killed by an elf zombie while frozen by a monster's gaze", "0xf88", 2662, "0x0", 7243, 1564092226, 1564100187, "Mal", "Law", "0x4", 8, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["FlamingKetchup/2019-07-25.22:08:03.ttyrec.bz2", 1, 22, 1649954022.628631, 8, 8, "3.6.3", 1254, 2, 4, 5, 0, 43, 1, 20190726, 20190725, 5, "Kni", "Hum", "Mal", "Law", "FlamingKetchup", "killed by an elf zombie while frozen by a monster's gaze", "0xf88", 2662, "0x0", 7243, 1564092226, 1564100187, "Mal", "Law", "0x4", 8, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["FlamingKetchup/2019-07-25.22:16:19.ttyrec.bz2", 2, 22, 1649954022.63341, 8, 8, "3.6.3", 1254, 2, 4, 5, 0, 43, 1, 20190726, 20190725, 5, "Kni", "Hum", "Mal", "Law", "FlamingKetchup", "killed by an elf zombie while frozen by a monster's gaze", "0xf88", 2662, "0x0", 7243, 1564092226, 1564100187, "Mal", "Law", "0x4", 8, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["aanderson/2019-07-21.15:10:27.ttyrec.bz2", 0, 22, 1649954022.644775, 9, 9, "3.6.3", 533, 0, 4, 4, 0, 20, 1, 20190729, 20190721, 5, "Wiz", "Elf", "Mal", "Cha", "aanderson", "killed by a wererat", "0xf88", 882, "0x0", 7995, 1563721850, 1564435194, "Mal", "Cha", "0x4", 9, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["aanderson/2019-07-29.21:13:49.ttyrec.bz2", 1, 22, 1649954022.649788, 9, 9, "3.6.3", 533, 0, 4, 4, 0, 20, 1, 20190729, 20190721, 5, "Wiz", "Elf", "Mal", "Cha", "aanderson", "killed by a wererat", "0xf88", 882, "0x0", 7995, 1563721850, 1564435194, "Mal", "Cha", "0x4", 9, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["FlamingKetchup/2019-07-30.16:06:23.ttyrec.bz2", 0, 22, 1649954022.640018, 10, 10, "3.6.3", 746, 2, 5, 6, 0, 43, 1, 20190730, 20190730, 5, "Val", "Hum", "Fem", "Law", "FlamingKetchup", "killed by a gnome", "0xfc0", 1875, "0x0", 1577, 1564502799, 1564504378, "Fem", "Law", "0x4", 10, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["aanderson/2019-08-03.16:56:46.ttyrec.bz2", 0, 22, 1649954022.655513, 11, 11, "3.6.3", 464, 0, 3, 5, 0, 25, 1, 20190803, 20190803, 5, "Wiz", "Gno", "Mal", "Neu", "aanderson", "killed by a homunculus while sleeping", "0xf88", 1140, "0x0", 764, 1564851435, 1564852201, "Mal", "Neu", "0x4", 11, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["aanderson/2019-08-04.09:40:55.ttyrec.bz2", 0, 22, 1649954022.673842, 13, 13, "3.6.3", 1035, 2, 6, 6, 0, 46, 1, 20190804, 20190804, 5, "Wiz", "Gno", "Mal", "Neu", "aanderson", "killed by a rothe", "0xe88", 1003, "0x0", 8447, 1564911667, 1564935411, "Mal", "Neu", "0x4", 13, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["aanderson/2019-08-04.15:56:08.ttyrec.bz2", 1, 22, 1649954022.678656, 13, 13, "3.6.3", 1035, 2, 6, 6, 0, 46, 1, 20190804, 20190804, 5, "Wiz", "Gno", "Mal", "Neu", "aanderson", "killed by a rothe", "0xe88", 1003, "0x0", 8447, 1564911667, 1564935411, "Mal", "Neu", "0x4", 13, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["aartist/2008-09-08.06:45:16.ttyrec.bz2", 0, 22, 1649954022.692907, 14, 14, "3.4.3", 537, 0, 2, 2, 31, 31, 0, 20080909, 20080908, 5, "Val", "Hum", "Fem", "Neu", "aartist", "quit", "0xfce", 1346, "0x0", 1149, 1220856342, 1220926886, "Fem", "Neu", "-1", 14, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["aartist/2008-09-09.02:02:56.ttyrec.bz2", 1, 22, 1649954022.699497, 14, 14, "3.4.3", 537, 0, 2, 2, 31, 31, 0, 20080909, 20080908, 5, "Val", "Hum", "Fem", "Neu", "aartist", "quit", "0xfce", 1346, "0x0", 1149, 1220856342, 1220926886, "Fem", "Neu", "-1", 14, "altorgtest", "altorgtest", "/path/to/altorg", 1],
["aartist/2008-09-09.02:21:28.ttyrec.bz2", 0, 22, 1649954022.704478, 15, 15, "3.4.3", 134, 0, 2, 2, 14, 15, 1, 20080909, 20080909, 5, "Sam", "Hum", "Mal", "Law", "aartist", "choked on a gunyoki", "0xf88", 251, "0x0", 373, 1220926913, 1220927313, "Mal", "Law", "-1", 15, "altorgtest", "altorgtest", "/path/to/altorg", 1]
]
9 changes: 9 additions & 0 deletions nle/tests/test_db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,3 +193,12 @@ def test_games(self, conn):
for gameid, _, _ in user_games["ccc"]:
for i, ttyrec in enumerate(conn.execute(cmd, (gameid,)).fetchall()):
assert ttyrec[1] == i

def test_version(self):
# Expect adding nledata to provide latest ttyrec_version
ttyrec_version = db.get_ttyrec_version("nletest")
assert ttyrec_version == nle.nethack.TTYREC_VERSION

# Expect adding altorg data additions to give version 1
ttyrec_version = db.get_ttyrec_version("altorgtest")
assert ttyrec_version == 1
8 changes: 5 additions & 3 deletions nle/tests/test_populate_db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
from test_db import conn # NOQA: F401
from test_db import mockdata # NOQA: F401

import nle
from nle import nethack

TTYRECS_TABLE_OFFSET = 0
GAMES_TABLE_OFFSET = 5
DATASETS_TABLE_OFFSET = 27 + GAMES_TABLE_OFFSET
ROOTS_TABLE_OFFSET = 3 + DATASETS_TABLE_OFFSET
ROOTS_TABLE_OFFSET = 2 + DATASETS_TABLE_OFFSET

TTYRECS_PATH_IDX = TTYRECS_TABLE_OFFSET + 0
TTYRECS_PART_IDX = TTYRECS_TABLE_OFFSET + 1
Expand All @@ -24,6 +24,7 @@
GAMES_ALIGN_IDX = GAMES_TABLE_OFFSET + 15
GAMES_NAME_IDX = GAMES_TABLE_OFFSET + 16
GAMES_DEATH_IDX = GAMES_TABLE_OFFSET + 17
TTYREC_VERSION_IDX = ROOTS_TABLE_OFFSET + 2

DATASETS_GAMEID_IDX = DATASETS_TABLE_OFFSET + 0

Expand Down Expand Up @@ -83,7 +84,7 @@ def test_dump_nle_db(self, conn): # NOQA: F811
".2.ttyrec%i.bz2",
".4.ttyrec%i.bz2",
]
endings = [e % nle.nethack.TTYREC_VERSION for e in endings]
endings = [e % nethack.TTYREC_VERSION for e in endings]
assert len(result) == 6

paths = []
Expand All @@ -100,5 +101,6 @@ def test_dump_nle_db(self, conn): # NOQA: F811
assert actual[GAMES_ALIGN_IDX] == "Neu"
assert actual[GAMES_NAME_IDX] == "Agent"
assert actual[GAMES_DEATH_IDX] == "escaped"
assert actual[TTYREC_VERSION_IDX] == nethack.TTYREC_VERSION

assert paths == sorted(paths)

0 comments on commit 037d554

Please sign in to comment.