Merge pull request #12 from shawngmc/develop

v0.0.21

.gitignore

@@ -342,3 +342,5 @@ $RECYCLE.BIN/
 
 # End of https://www.toptal.com/developers/gitignore/api/python,node,linux,windows
 
+# Ignore output folder used for testing
+out/

.pylintrc

@@ -0,0 +1,4 @@
+[MESSAGES CONTROL]
+# W1202, W1203Allow f-string log messages
+enable=W1202
+disable=W1203

.vscode/launch.json

@@ -13,15 +13,29 @@
             "cwd": "${workspaceFolder}/src/gex",
             "justMyCode": true,
             "args": [
-                "postprocess", 
-                "steam", 
+                "tasks", 
                 "extract", 
                 "--task", 
                 "cbeub", 
-                "--srcdir", 
-                "\"C:\\Program Files (x86)\\Steam\\steamapps\\common\\CBEUB\"", 
                 "--destdir", 
-                "\"C:\\Users\\shawn\\Desktop\\rom playground\\gex_testing\\cbeub\""
+                "\"${workspaceFolder}\\out\\cbeub\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing Collection of SaGa",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "saga", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\saga\""
             ]
         },
         {
@@ -33,15 +47,12 @@
             "cwd": "${workspaceFolder}/src/gex",
             "justMyCode": true,
             "args": [
-                "postprocess", 
-                "steam", 
+                "tasks", 
                 "extract", 
                 "--task", 
                 "cfc", 
-                "--srcdir", 
-                "\"C:\\Program Files (x86)\\Steam\\steamapps\\common\\CAPCOM FIGHTING COLLECTION\"", 
                 "--destdir", 
-                "\"C:\\Users\\shawn\\Desktop\\rom playground\\gex_testing\\cfc\""
+                "\"${workspaceFolder}\\out\\cfc\""
             ]
         },
         {
@@ -53,15 +64,12 @@
             "cwd": "${workspaceFolder}/src/gex",
             "justMyCode": true,
             "args": [
-                "postprocess", 
-                "steam", 
+                "tasks", 
                 "extract", 
                 "--task", 
                 "sf30ac", 
-                "--srcdir", 
-                "\"C:\\Program Files (x86)\\Steam\\steamapps\\common\\Street Fighter 30th Anniversary Collection\"", 
                 "--destdir", 
-                "\"C:\\Users\\shawn\\Desktop\\rom playground\\gex_testing\\sf30ac\""
+                "\"${workspaceFolder}\\out\\sf30ac\""
             ]
         },
         {
@@ -73,15 +81,12 @@
             "cwd": "${workspaceFolder}/src/gex",
             "justMyCode": true,
             "args": [
-                "postprocess", 
-                "steam", 
+                "tasks", 
                 "extract", 
                 "--task", 
                 "cas1_old", 
-                "--srcdir", 
-                "\"C:\\Program Files (x86)\\Steam\\steamapps\\content\"", 
                 "--destdir", 
-                "\"C:\\Users\\shawn\\Desktop\\rom playground\\gex_testing\\cas1_old\""
+                "\"${workspaceFolder}\\out\\cas1_old\""
             ]
         },
         {
@@ -93,25 +98,132 @@
             "cwd": "${workspaceFolder}/src/gex",
             "justMyCode": true,
             "args": [
-                "postprocess", 
-                "steam", 
+                "tasks", 
                 "extract", 
                 "--task", 
                 "genesis", 
-                "--srcdir", 
-                "\"C:\\Program Files (x86)\\Steam\\steamapps\\common\\Sega Classics\"",
                 "--destdir", 
-                "\"C:\\Users\\shawn\\Desktop\\rom playground\\gex_testing\\genesis\""
+                "\"${workspaceFolder}\\out\\genesis\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing BlizzArcade",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "blizzarcade", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\blizzarcade\""
             ]
         },
         {
-            "name": "Python: Deshuffle Perf Comparison",
+            "name": "Python: Toolbox - Testing MMLC1",
             "type": "python",
             "request": "launch",
-            "program": "deshuffle_compare.py",
+            "program": "toolbox.py",
             "console": "integratedTerminal",
-            "cwd": "${workspaceFolder}/testing",
-            "justMyCode": true
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "mmlc1", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\mmlc1\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing MMXLC1",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "mmxlc1", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\mmxlc1\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing Bubsy",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "bubsy", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\bubsy\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing Double Dragon Trilogy",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "ddragontrilogy", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\ddragontrilogy\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing Disney Afternoon Collection",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "disneyac", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\disneyac\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing SADXGG",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "sadxgg", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\sadxgg\""
+            ]
         }
     ]
 }

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "python.linting.pylintEnabled": true,
+    "python.linting.enabled": true
+}

CHANGELOG.md

@@ -1,3 +1,23 @@
+# 0.0.21
+- Added Games/Collections:
+  - Blizzard Arcade Collection
+  - Bubsy Two-Fur
+  - Collection of SaGa/Final Fantasy Legend
+  - Disney Afternoon Collection
+  - Double Dragon Trilogy
+  - Mega Man Legacy Collection 1
+  - Mega Man X Legacy Collection 1
+  - Sonic Adventure DX (Hidden Game Gear games)
+- Improvements:
+  - Moved 'postprocess steam' commands to more generic 'tasks' command
+    This is one of the main purposes of this program, and not Steam specific - GOG, etc. should also be considered
+  - Added default input folders for Tasks
+  - Fixing tables in task details
+- Internal:
+  - Test configs for development should now work on any machine
+  - Tasks are now slightly more defined using a class
+  - First major lint/code style effort; nowhere near done, but much more idiomatic
+
 # 0.0.20
 - Added Games/Collections:
   - Sega Genesis and Mega Drive Collection

DEVELOPMENT.md

@@ -34,4 +34,11 @@
 
 ### Manual
 1. Distribute via twine ```twine upload dist/*```
-2. Clean out dist dir
+2. Clean out dist dir
+
+
+## Special Topics
+### Markdown Tables
+In the task details, markdown tables behave oddly:
+* Rows are not resized automatically
+* Every line needs 1-2 spaces at the end

README.md

@@ -60,13 +60,51 @@ I prefer not to be artificially locked to specific platforms - why play a 25 yea
 ### Why not just download the ROMs?
 It is illegal. One can discuss economic moral implications all day, but at the end of the day, it's theft. As a software developer myself, I don't want my code stolen - so I can't claim a moral high ground there. Equally importantly, showing classic game owners that the market will support rereleases increases the liklihood of rereleases, and I want people who can't/won't download ROMs to enjoy these titles as well.
 
+### This MAME ROM is actually XXXXYY, not XXXXZZ - why do you have it that way?
+The goal of this tool is not to create a perfect ROM set for the latest MAME, because it can't. A few factors go into what version of MAME we target:
+
+* What version of MAME did the rerelease use?
+* Are there missing files, like encryption keys or dl-1425.bin, that force us to target a different version of MAME to make a ROM that is usable out of the box?
+* When did MAME split specific collections?
+* What are file CRCs?
+* Did the rerelease modify files for legal/moral reasons?
+* How can we make a playable ROM?
+
+For example, let's use Street Fighter 30th Anniversary Collection's copy of Street Fighter Alpha 2. It's technically both sfa2u and sfa2ur1, and neither! Here's why, using Arcade Italia's [SFA2U](http://adb.arcadeitalia.net/dettaglio_mame.php?game_name=sfa2u) and [SFA2UR1](http://adb.arcadeitalia.net/dettaglio_mame.php?game_name=sfa2ur1) as a source:
+
+1. The primary difference between these two ROMs is maincpu, like sz2u.03.
+2. At first glance, this looks like sfa2ur1, because the sz2u.03 we create has a CRC of 84a09006. Case closed, right?
+3. Well, not so fast. If we make a sfa2ur1 or sfa2u zip file, neither are natively playable in MAME 0.246, because we don't have the sfa2u.key or dl-1425.bin.
+4. The standard solution here - since this project will not add copyrighted material not in the source - is to target an older version of MAME, 0.139 (aka Mame 2010 in RetroArch). Except... sfa2ur1 didn't exist until MAME 0.141!
+5. We then look at sfa2u before the split. Lo and behold, sz2u.03 has the CRC of 84a90006. OK, so it's sfa2u, right?
+6. Kinda. The gfx CRCs don't match EITHER ROM. This is most likely because Capcom modified the graphics to remove copyrighted or offensive content. However, there's no MAME clone with the gfx CRCs we have.
+7. Since our goal is to make a playable ROM, not a perfect one, we can test the ROM in MAME 0.139. It works, so we'll call it sfa2u, but make a note in the documentation for that command.
+
+These are noted in the documentation for each script. If you think a ROM is misidentified, please:
+
+1. Search for a closed issue - these will likely be brought up multiple times!
+2. If none exists, use a source like [Arcade Italia](http://adb.arcadeitalia.net/) to verify. Make sure you look at the original file we create, and watch for the following things:
+  * The MAME version we target
+  * Any missing files
+  * When a ROM was split
+  * CRCs for every file
+3. If you are solidly convinced that there's a misnaming, open a ticket and we'll take a look.
+
 
 ## Collections
 
-**Collection**                             | **Status**     | **Notes**                                                               
----------------------------------|-------------|---------------------------------------------------------------------
- **Capcom Arcade Stadium 1**                    | 95%    | Requires Steam depot downloading, a couple shaky ROMs...
- **Capcom Beat 'Em Up Bundle**                  | 95%    | 6/7 playable on some version of MAME, but wof/wofj missing audiocpu data
- **Capcom Fighting Collection**                 | 90%    | CPS2 is semi-standard. No Enc keys present. CP3 game is a curveball!
- **Street Fighter 30th Anniversary Collection** | 100%   | All games are playable with some version of MAME!
- **Sega Genesis and Mega Drive Collection**     | 90%    | Some compressed variants not yet extracted
+**Collection**                             | **Status**     | **Notes**           
+--------------------------------------------|-------------|---------------------------------------------------------------------  
+ **Blizzard Arcade Collection**                 | 100%   |   
+ **Bubsy Two-Fur**                              | 100%   |   
+ **Capcom Arcade Stadium 1**                    | 95%    | Requires Steam depot downloading, a couple shaky ROMs...  
+ **Capcom Beat 'Em Up Bundle**                  | 95%    | 6/7 playable on some version of MAME, but wof/wofj missing audiocpu data  
+ **Capcom Fighting Collection**                 | 90%    | CPS2 is semi-standard. No Enc keys present. CP3 game is a curveball!  
+ **Collection of SaGa/Final Fantasy Legend**    | 100%   |  
+ **Disney Afternoon Collection**                | 100%   |  
+ **Double Dragon Trilogy**                      | 100%   |  
+ **Mega Man Legacy Collection 1**               | 100%   |  
+ **Mega Man X Legacy Collection 1**             | 75%    | X4 doesn't appear to be ROM based  
+ **Sega Genesis and Mega Drive Collection**     | 90%    | Some compressed variants not yet extracted  
+ **Sonic Adventure DX (Hidden Game Gear games)**| 100%   | This is only the Game Gear games - SADX itself can not be made into a ROM/ISO!
+ **Street Fighter 30th Anniversary Collection** | 100%   | All games are playable with some version of MAME!