diff --git a/.github/workflows/linux_test.yml b/.github/workflows/linux_test.yml index 1668bec..5e7daf2 100644 --- a/.github/workflows/linux_test.yml +++ b/.github/workflows/linux_test.yml @@ -21,23 +21,3 @@ jobs: - name: Test AutoROM run: | ./scripts/ci-test.sh - - decrypt-roms-test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Decrypt Roms.tar.gz - env: - ROMS_B64_LINK: ${{ secrets.ROMS_B64_LINK }} - run: | - wget "$ROMS_B64_LINK" - base64 Roms.tar.gz.b64 --decode &> Roms.tar.gz - pip install packages/* - AutoROM --accept-license --source-file Roms.tar.gz diff --git a/README.md b/README.md index 9419ebd..7ffd8fb 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,11 @@ AutoROM automatically installs Atari ROM files for ALE-Py (which Gymnasium Depends on) and multi-agent-ALE (which PettingZoo depends on, but will replaced by ALE-Py in the future). AutoROM automatically downloads the needed Atari ROMs from ROM hosting websites into the ALE-Py folder and Multi-Agent-ALE-py folder in a very simple manner: - ``` pip install autorom AutoROM ``` -> IMPORTANT: We do not have official support for Windows. However, if you are on Windows and encounter a DLL import error, you may need to install [OpenSSL v1.1.1S](https://slproweb.com/products/Win32OpenSSL.html). -If you encounter problems with installation due to your network blocking torrenting, it is possible to install the ROMs using a pre-downloaded roms.tar.gz file. -``` -AutoROM --source-file /path/to/roms.tar.gz -``` To specify a specific installation directory for your ROMs, use the `--install-dir` command line flag. ``` AutoROM --install-dir /path/to/install diff --git a/packages/AutoROM.accept-rom-license/pyproject.toml b/packages/AutoROM.accept-rom-license/pyproject.toml index 0ed6d3c..79f3c02 100644 --- a/packages/AutoROM.accept-rom-license/pyproject.toml +++ b/packages/AutoROM.accept-rom-license/pyproject.toml @@ -5,7 +5,6 @@ requires = [ "wheel", "requests", "tqdm", - "libtorrent", "click", "importlib-resources; python_version < '3.9'" ] diff --git a/requirements.txt b/requirements.txt index bfc3eee..99431da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ click requests -libtorrent importlib-resources; python_version < '3.9' diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index de5e525..68c6892 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -22,11 +22,9 @@ test_autorom() { # Get install flag local install_to_pkgs=false - local pretorrented=false - while getopts 'ip' opt; do + while getopts 'i' opt; do case $opt in i) install_to_pkgs=true ;; - p) pretorrented=true ;; esac done @@ -36,9 +34,6 @@ test_autorom() { if [ "$install_to_pkgs" = true ]; then # conditionally install to packages AutoROM --accept-license - elif [ "$pretorrented" = true ]; then - # conditionally test using pretorrented - AutoROM --accept-license --source-file ../scripts/Roms.tar.gz else # generic install AutoROM --accept-license --install-dir . @@ -80,13 +75,3 @@ pip install --find-links dist/ --no-cache-dir AutoROM[accept-rom-license] test_autorom test_cleanup echo "::endgroup::" - -# Test installing using pre-torrented tar -echo "::group::Test AutoROM[accept-rom-license]" -test_init -pip install --find-links dist/ --no-cache-dir AutoROM -python ./scripts/torrent_tar.py - -test_autorom -p -test_cleanup -echo "::endgroup::" diff --git a/src/AutoROM.py b/src/AutoROM.py index 49c338f..92e727c 100644 --- a/src/AutoROM.py +++ b/src/AutoROM.py @@ -1,29 +1,18 @@ #!/usr/bin/env python3 +import base64 import hashlib import io import os import pathlib import sys import tarfile -import time import warnings +from collections import namedtuple +from typing import Dict import click import requests -if os.name == "nt": - try: - import libtorrent as lt - except ImportError as e: - raise ImportError( - "It seems that you are trying to install the Atari ROMs on Windows. While this is not supported, the DLL error can be solved by installing the OpenSSL DLLs from: https://slproweb.com/products/Win32OpenSSL.html" - ) from e -else: - import libtorrent as lt - -from collections import namedtuple -from typing import Dict - if sys.version_info < (3, 9): import importlib_resources as resources else: @@ -152,74 +141,20 @@ } -def torrent_tar(): - # specify the save path - save_path = os.path.dirname(__file__) - save_file = os.path.join(save_path, "./Roms.tar.gz") - - # magnet uri - uri = "magnet:?xt=urn:btih:a606d1dabf28e794cbc0f88f10d0b8225dc854b4&dn=Roms.tar.gz&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=http%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2F9.rarbg.com%3A2810%2Fannounce&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A6969%2Fannounce&tr=http%3A%2F%2Ftracker.openbittorrent.com%3A80%2Fannounce&tr=udp%3A%2F%2Fopentracker.i2p.rocks%3A6969%2Fannounce&tr=https%3A%2F%2Fopentracker.i2p.rocks%3A443%2Fannounce&tr=udp%3A%2F%2Fopen.stealth.si%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.torrent.eu.org%3A451%2Fannounce&tr=udp%3A%2F%2Ftracker2.dler.org%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.tiny-vps.com%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.moeking.me%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.dler.org%3A6969%2Fannounce&tr=udp%3A%2F%2Fpublic.tracker.vraphim.com%3A6969%2Fannounce&tr=udp%3A%2F%2Fp4p.arenabg.com%3A1337%2Fannounce&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce&tr=udp%3A%2F%2Fmovies.zsw.ca%3A6969%2Fannounce&tr=udp%3A%2F%2Fipv4.tracker.harry.lu%3A80%2Fannounce&tr=udp%3A%2F%2Ffe.dealclub.de%3A6969%2Fannounce&tr=udp%3A%2F%2Fexplodie.org%3A6969%2Fannounce&tr=udp%3A%2F%2Fexodus.desync.com%3A6969%2Fannounce&tr=udp%3A%2F%2Fbt2.archive.org%3A6969%2Fannounce&tr=udp%3A%2F%2Fbt1.archive.org%3A6969%2Fannounce&tr=udp%3A%2F%2F6ahddutb1ucc3cp.ru%3A6969%2Fannounce&tr=https%3A%2F%2Ftracker.nanoha.org%3A443%2Fannounce&tr=https%3A%2F%2Ftracker.lilithraws.org%3A443%2Fannounce&tr=https%3A%2F%2Ftr.burnabyhighstar.com%3A443%2Fannounce&tr=http%3A%2F%2Fvps02.net.orel.ru%3A80%2Fannounce&tr=http%3A%2F%2Ftracker2.dler.org%3A80%2Fannounce&tr=http%3A%2F%2Ftracker.mywaifu.best%3A6969%2Fannounce&tr=http%3A%2F%2Ftracker.files.fm%3A6969%2Fannounce&tr=http%3A%2F%2Ftracker.dler.org%3A6969%2Fannounce&tr=http%3A%2F%2Ft.overflow.biz%3A6969%2Fannounce&tr=udp%3A%2F%2Fzecircle.xyz%3A6969%2Fannounce&tr=udp%3A%2F%2Fyahor.ftp.sh%3A6969%2Fannounce&tr=udp%3A%2F%2Fvibe.sleepyinternetfun.xyz%3A1738%2Fannounce&tr=udp%3A%2F%2Fuploads.gamecoast.net%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker1.bt.moack.co.kr%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.theoks.net%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.tcp.exchange%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.swateam.org.uk%3A2710%2Fannounce&tr=udp%3A%2F%2Ftracker.srv00.com%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.skyts.net%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.publictracker.xyz%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.pomf.se%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.openbtba.com%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.monitorit4.me%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.lelux.fi%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.leech.ie%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.joybomb.tw%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.jonaslsa.com%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.filemail.com%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.ddunlimited.net%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.bitsearch.to%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.auctor.tv%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.artixlinux.org%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.army%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.altrosky.nl%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.4.babico.name.tr%3A3131%2Fannounce&tr=udp%3A%2F%2Ftracker-udp.gbitt.info%3A80%2Fannounce&tr=udp%3A%2F%2Ftorrents.artixlinux.org%3A6969%2Fannounce&tr=udp%3A%2F%2Ftorrentclub.space%3A6969%2Fannounce&tr=udp%3A%2F%2Fthouvenin.cloud%3A6969%2Fannounce&tr=udp%3A%2F%2Ftamas3.ynh.fr%3A6969%2Fannounce&tr=udp%3A%2F%2Fsmtp-relay.odysseylabel.com.au%3A6969%2Fannounce&tr=udp%3A%2F%2Fsanincode.com%3A6969%2Fannounce&tr=udp%3A%2F%2Frun.publictracker.xyz%3A6969%2Fannounce&tr=udp%3A%2F%2Frun-2.publictracker.xyz%3A6969%2Fannounce&tr=udp%3A%2F%2Frep-art.ynh.fr%3A6969%2Fannounce&tr=udp%3A%2F%2Frekcart.duckdns.org%3A15480%2Fannounce&tr=udp%3A%2F%2Fqtstm32fan.ru%3A6969%2Fannounce&tr=udp%3A%2F%2Fpublic.publictracker.xyz%3A6969%2Fannounce&tr=udp%3A%2F%2Fpsyco.fr%3A6969%2Fannounce&tr=udp%3A%2F%2Fopen.tracker.ink%3A6969%2Fannounce&tr=udp%3A%2F%2Fopen.free-tracker.ga%3A6969%2Fannounce&tr=udp%3A%2F%2Fopen.dstud.io%3A6969%2Fannounce&tr=udp%3A%2F%2Fnew-line.net%3A6969%2Fannounce&tr=udp%3A%2F%2Fmoonburrow.club%3A6969%2Fannounce&tr=udp%3A%2F%2Fmirror.aptus.co.tz%3A6969%2Fannounce&tr=udp%3A%2F%2Fmail.zasaonsk.ga%3A6969%2Fannounce&tr=udp%3A%2F%2Fmail.artixlinux.org%3A6969%2Fannounce&tr=udp%3A%2F%2Fmadiator.com%3A6969%2Fannounce&tr=udp%3A%2F%2Fleefafa.tk%3A6969%2Fannounce&tr=udp%3A%2F%2Flaze.cc%3A6969%2Fannounce&tr=udp%3A%2F%2Fkokodayo.site%3A6969%2Fannounce&tr=udp%3A%2F%2Fkeke.re%3A6969%2Fannounce&tr=udp%3A%2F%2Fhtz3.noho.st%3A6969%2Fannounce&tr=udp%3A%2F%2Ffh2.cmp-gaming.com%3A6969%2Fannounce&tr=udp%3A%2F%2Ff1sh.de%3A6969%2Fannounce&tr=udp%3A%2F%2Fepider.me%3A6969%2Fannounce&tr=udp%3A%2F%2Felementsbrowser.com%3A6969%2Fannounce&tr=udp%3A%2F%2Fdownload.nerocloud.me%3A6969%2Fannounce&tr=udp%3A%2F%2Fcutscloud.duckdns.org%3A6969%2Fannounce&tr=udp%3A%2F%2Fconcen.org%3A6969%2Fannounce&tr=udp%3A%2F%2Fchouchou.top%3A8080%2Fannounce&tr=udp%3A%2F%2Fcarr.codes%3A6969%2Fannounce&tr=udp%3A%2F%2Fcamera.lei001.com%3A6969%2Fannounce&tr=udp%3A%2F%2Fbuddyfly.top%3A6969%2Fannounce&tr=udp%3A%2F%2Fbubu.mapfactor.com%3A6969%2Fannounce&tr=udp%3A%2F%2Fbt.ktrackers.com%3A6666%2Fannounce&tr=udp%3A%2F%2Fblack-bird.ynh.fr%3A6969%2Fannounce&tr=udp%3A%2F%2Fben.kerbertools.xyz%3A6969%2Fannounce&tr=udp%3A%2F%2Fbananas.space%3A6969%2Fannounce&tr=udp%3A%2F%2Fastrr.ru%3A6969%2Fannounce&tr=udp%3A%2F%2Fapp.icon256.com%3A8000%2Fannounce&tr=udp%3A%2F%2Fadmin.videoenpoche.info%3A6969%2Fannounce&tr=udp%3A%2F%2Fadmin.52ywp.com%3A6969%2Fannounce&tr=udp%3A%2F%2Faarsen.me%3A6969%2Fannounce&tr=udp%3A%2F%2F960303.xyz%3A6969%2Fannounce&tr=https%3A%2F%2Fxtremex.herokuapp.com%3A443%2Fannounce&tr=https%3A%2F%2Ftracker2.ctix.cn%3A443%2Fannounce&tr=https%3A%2F%2Ftracker1.520.jp%3A443%2Fannounce&tr=https%3A%2F%2Ftracker.tamersunion.org%3A443%2Fannounce&tr=https%3A%2F%2Ftracker.kuroy.me%3A443%2Fannounce&tr=https%3A%2F%2Ftracker.gbitt.info%3A443%2Fannounce&tr=https%3A%2F%2Ftracker.foreverpirates.co%3A443%2Fannounce&tr=https%3A%2F%2Ftracker.expli.top%3A443%2Fannounce&tr=https%3A%2F%2Ftr.abir.ga%3A443%2Fannounce&tr=https%3A%2F%2Ftr.abiir.top%3A443%2Fannounce&tr=https%3A%2F%2F1337.abcvg.info%3A443%2Fannounce&tr=http%3A%2F%2Fwepzone.net%3A6969%2Fannounce&tr=http%3A%2F%2Ftracker4.itzmx.com%3A2710%2Fannounce&tr=http%3A%2F%2Ftracker3.itzmx.com%3A6961%2Fannounce&tr=http%3A%2F%2Ftracker3.ctix.cn%3A8080%2Fannounce&tr=http%3A%2F%2Ftracker1.itzmx.com%3A8080%2Fannounce&tr=http%3A%2F%2Ftracker1.bt.moack.co.kr%3A80%2Fannounce&tr=http%3A%2F%2Ftracker.skyts.net%3A6969%2Fannounce&tr=http%3A%2F%2Ftracker.lelux.fi%3A80%2Fannounce&tr=http%3A%2F%2Ftracker.gbitt.info%3A80%2Fannounce&tr=http%3A%2F%2Ftracker.edkj.club%3A6969%2Fannounce&tr=http%3A%2F%2Ftracker.bt4g.com%3A2095%2Fannounce&tr=http%3A%2F%2Ftorrenttracker.nwc.acsalaska.net%3A6969%2Fannounce&tr=http%3A%2F%2Ft.acg.rip%3A6699%2Fannounce&tr=http%3A%2F%2Fopen.tracker.ink%3A6969%2Fannounce&tr=http%3A%2F%2Fopen.acgnxtracker.com%3A80%2Fannounce&tr=http%3A%2F%2Fjp.moeweb.pw%3A6969%2Fannounce&tr=http%3A%2F%2Fincine.ru%3A6969%2Fannounce&tr=http%3A%2F%2Ffxtt.ru%3A80%2Fannounce&tr=http%3A%2F%2Fbt.okmp3.ru%3A2710%2Fannounce&tr=http%3A%2F%2F1337.abcvg.info%3A80%2Fannounce" - - # libtorrent params - ses = lt.session() - params = lt.parse_magnet_uri(uri) - params.save_path = save_path - handle: lt.torrent_handle = ses.add_torrent(params) - - # download roms as long as state is not seeding - timeit = 0 - while handle.status().state not in {4, 5}: - if timeit >= 360: - raise RuntimeError( - "Terminating attempt to download ROMs after 180 seconds, this has failed.\n" - "Try these lines instead:" - "-------------------------\n" - "wget https://gist.githubusercontent.com/jjshoots/61b22aefce4456920ba99f2c36906eda/raw/00046ac3403768bfe45857610a3d333b8e35e026/Roms.tar.gz.b64\n" - "base64 Roms.tar.gz.b64 --decode &> Roms.tar.gz\n" - "AutoROM --accept-license --source-file Roms.tar.gz\n" - "-------------------------" - ) +def download_tar(): + # use requests to download the base64 file + url = "https://gist.githubusercontent.com/jjshoots/61b22aefce4456920ba99f2c36906eda/raw/00046ac3403768bfe45857610a3d333b8e35e026/Roms.tar.gz.b64" + r = requests.get(url, allow_redirects=False) - if timeit % 5 == 0: - if timeit >= 180: - print( - "Have been attempting to download for more than 180 seconds, consider terminating?", - file=sys.stderr, - ) - print( - "Optionally, run these lines:\n" - "-------------------------\n" - "wget https://gist.githubusercontent.com/jjshoots/61b22aefce4456920ba99f2c36906eda/raw/00046ac3403768bfe45857610a3d333b8e35e026/Roms.tar.gz.b64\n" - "base64 Roms.tar.gz.b64 --decode &> Roms.tar.gz\n", - "AutoROM --accept-license --source-file Roms.tar.gz\n" - "-------------------------", - file=sys.stderr, - ) - - status: lt.torrent_status = handle.status() - print( - f"time={timeit} / 180 seconds - Trying to download atari roms\n" - f"\tcurrent status={status_meaning.get(status.state, 'unknown')} ({status.state})\n" - f"\ttotal downloaded bytes={status.total_download}\n" - f"\ttotal payload download={status.total_payload_download}\n" - f"\ttotal failed bytes={status.total_failed_bytes}", - file=sys.stderr, - ) + # decode the b64 into the tar.gz + tar_gz = base64.b64decode(r.content) - # some sleep helps - time.sleep(1.0) - timeit += 1 - - # seed for 20 seconds to help the network - if handle.status().state in {4, 5}: - print( - "Download complete, seeding for 20 seconds to assist torrent network.", - file=sys.stderr, - ) - time.sleep(20.0) - print("Seeding completed.", file=sys.stderr) + # save the tar.gz + save_path = os.path.dirname(__file__) + save_path = os.path.join(save_path, "./Roms.tar.gz") + open(save_path, "wb").write(tar_gz) - return save_file + return save_path def verify_installation(package, checksum_keys): @@ -372,7 +307,7 @@ def main(accept_license, source_file, install_dir, quiet): ): return - with open(torrent_tar() if source_file is None else source_file, "rb") as fh: + with open(download_tar() if source_file is None else source_file, "rb") as fh: buffer = io.BytesIO(fh.read()) extract_roms_from_tar(buffer, packages, checksum_map, quiet) diff --git a/version.txt b/version.txt index d1d899f..a918a2a 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.5 +0.6.0