Merge branch 'main' into main

.github/workflows/unittests.yml

@@ -54,9 +54,9 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pytest pytest-subtests
+        pip install pytest pytest-subtests pytest-xdist
         python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt"
         python Launcher.py --update_settings  # make sure host.yaml exists for tests
     - name: Unittests
       run: |
-        pytest
+        pytest -n auto

.gitignore

@@ -27,16 +27,20 @@
 *.archipelago
 *.apsave
 *.BIN
+*.puml
 
 setups
 build
 bundle/components.wxs
 dist
+/prof/
 README.html
 .vs/
 EnemizerCLI/
 /Players/
 /SNI/
+/sni-*/
+/appimagetool*
 /host.yaml
 /options.yaml
 /config.yaml
@@ -139,6 +143,7 @@ ipython_config.py
 .venv*
 env/
 venv/
+/venv*/
 ENV/
 env.bak/
 venv.bak/

BaseClasses.py

@@ -8,6 +8,7 @@
 import typing  # this can go away when Python 3.8 support is dropped
 from argparse import Namespace
 from collections import ChainMap, Counter, deque
+from collections.abc import Collection
 from enum import IntEnum, IntFlag
 from typing import Any, Callable, Dict, Iterable, Iterator, List, NamedTuple, Optional, Set, Tuple, TypedDict, Union, \
     Type, ClassVar
@@ -202,14 +203,7 @@ def add_group(self, name: str, game: str, players: Set[int] = frozenset()) -> Tu
         self.player_types[new_id] = NetUtils.SlotType.group
         self._region_cache[new_id] = {}
         world_type = AutoWorld.AutoWorldRegister.world_types[game]
-        for option_key, option in world_type.option_definitions.items():
-            getattr(self, option_key)[new_id] = option(option.default)
-        for option_key, option in Options.common_options.items():
-            getattr(self, option_key)[new_id] = option(option.default)
-        for option_key, option in Options.per_game_common_options.items():
-            getattr(self, option_key)[new_id] = option(option.default)
-
-        self.worlds[new_id] = world_type(self, new_id)
+        self.worlds[new_id] = world_type.create_group(self, new_id, players)
         self.worlds[new_id].collect_item = classmethod(AutoWorld.World.collect_item).__get__(self.worlds[new_id])
         self.player_name[new_id] = name
 
@@ -232,25 +226,24 @@ def set_seed(self, seed: Optional[int] = None, secure: bool = False, name: Optio
                                  range(1, self.players + 1)}
 
     def set_options(self, args: Namespace) -> None:
-        for option_key in Options.common_options:
-            setattr(self, option_key, getattr(args, option_key, {}))
-        for option_key in Options.per_game_common_options:
-            setattr(self, option_key, getattr(args, option_key, {}))
-
         for player in self.player_ids:
             self.custom_data[player] = {}
             world_type = AutoWorld.AutoWorldRegister.world_types[self.game[player]]
-            for option_key in world_type.option_definitions:
-                setattr(self, option_key, getattr(args, option_key, {}))
-
             self.worlds[player] = world_type(self, player)
             self.worlds[player].random = self.per_slot_randoms[player]
+            for option_key in world_type.options_dataclass.type_hints:
+                option_values = getattr(args, option_key, {})
+                setattr(self, option_key, option_values)
+                # TODO - remove this loop once all worlds use options dataclasses
+            options_dataclass: typing.Type[Options.PerGameCommonOptions] = self.worlds[player].options_dataclass
+            self.worlds[player].options = options_dataclass(**{option_key: getattr(args, option_key)[player]
+                                                               for option_key in options_dataclass.type_hints})
 
     def set_item_links(self):
         item_links = {}
         replacement_prio = [False, True, None]
         for player in self.player_ids:
-            for item_link in self.item_links[player].value:
+            for item_link in self.worlds[player].options.item_links.value:
                 if item_link["name"] in item_links:
                     if item_links[item_link["name"]]["game"] != self.game[player]:
                         raise Exception(f"Cannot ItemLink across games. Link: {item_link['name']}")
@@ -305,14 +298,6 @@ def set_item_links(self):
             group["non_local_items"] = item_link["non_local_items"]
             group["link_replacement"] = replacement_prio[item_link["link_replacement"]]
 
-    # intended for unittests
-    def set_default_common_options(self):
-        for option_key, option in Options.common_options.items():
-            setattr(self, option_key, {player_id: option(option.default) for player_id in self.player_ids})
-        for option_key, option in Options.per_game_common_options.items():
-            setattr(self, option_key, {player_id: option(option.default) for player_id in self.player_ids})
-        self.state = CollectionState(self)
-
     def secure(self):
         self.random = ThreadBarrierProxy(secrets.SystemRandom())
         self.is_race = True
@@ -364,7 +349,7 @@ def _recache(self):
             for r_location in region.locations:
                 self._location_cache[r_location.name, player] = r_location
 
-    def get_regions(self, player=None):
+    def get_regions(self, player: Optional[int] = None) -> Collection[Region]:
         return self.regions if player is None else self._region_cache[player].values()
 
     def get_region(self, regionname: str, player: int) -> Region:
@@ -869,31 +854,31 @@ def add_locations(self, locations: Dict[str, Optional[int]],
         """
         Adds locations to the Region object, where location_type is your Location class and locations is a dict of
         location names to address.
-        
+
         :param locations: dictionary of locations to be created and added to this Region `{name: ID}`
         :param location_type: Location class to be used to create the locations with"""
         if location_type is None:
             location_type = Location
         for location, address in locations.items():
             self.locations.append(location_type(self.player, location, address, self))
-    
+
     def connect(self, connecting_region: Region, name: Optional[str] = None,
                 rule: Optional[Callable[[CollectionState], bool]] = None) -> None:
         """
         Connects this Region to another Region, placing the provided rule on the connection.
-        
+
         :param connecting_region: Region object to connect to path is `self -> exiting_region`
         :param name: name of the connection being created
         :param rule: callable to determine access of this connection to go from self to the exiting_region"""
         exit_ = self.create_exit(name if name else f"{self.name} -> {connecting_region.name}")
         if rule:
             exit_.access_rule = rule
         exit_.connect(connecting_region)
-    
+
     def create_exit(self, name: str) -> Entrance:
         """
         Creates and returns an Entrance object as an exit of this region.
-        
+
         :param name: name of the Entrance being created
         """
         exit_ = self.entrance_type(self.player, name, self)
@@ -1263,7 +1248,7 @@ def get_path(state: CollectionState, region: Region) -> List[Union[Tuple[str, st
 
     def to_file(self, filename: str) -> None:
         def write_option(option_key: str, option_obj: Options.AssembleOptions) -> None:
-            res = getattr(self.multiworld, option_key)[player]
+            res = getattr(self.multiworld.worlds[player].options, option_key)
             display_name = getattr(option_obj, "display_name", option_key)
             outfile.write(f"{display_name + ':':33}{res.current_option_name}\n")
 
@@ -1281,8 +1266,7 @@ def write_option(option_key: str, option_obj: Options.AssembleOptions) -> None:
                     outfile.write('\nPlayer %d: %s\n' % (player, self.multiworld.get_player_name(player)))
                 outfile.write('Game:                            %s\n' % self.multiworld.game[player])
 
-                options = ChainMap(Options.per_game_common_options, self.multiworld.worlds[player].option_definitions)
-                for f_option, option in options.items():
+                for f_option, option in self.multiworld.worlds[player].options_dataclass.type_hints.items():
                     write_option(f_option, option)
 
                 AutoWorld.call_single(self.multiworld, "write_spoiler_header", player, outfile)

BizHawkClient.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import ModuleUpdate
+ModuleUpdate.update()
+
+from worlds._bizhawk.context import launch
+
+if __name__ == "__main__":
+    launch()

CommonClient.py

@@ -1,4 +1,6 @@
 from __future__ import annotations
+
+import copy
 import logging
 import asyncio
 import urllib.parse
@@ -242,6 +244,7 @@ def __init__(self, server_address: typing.Optional[str], password: typing.Option
         self.watcher_event = asyncio.Event()
 
         self.jsontotextparser = JSONtoTextParser(self)
+        self.rawjsontotextparser = RawJSONtoTextParser(self)
         self.update_data_package(network_data_package)
 
         # execution
@@ -377,10 +380,13 @@ def on_print(self, args: dict):
 
     def on_print_json(self, args: dict):
         if self.ui:
-            self.ui.print_json(args["data"])
-        else:
-            text = self.jsontotextparser(args["data"])
-            logger.info(text)
+            # send copy to UI
+            self.ui.print_json(copy.deepcopy(args["data"]))
+
+        logging.getLogger("FileLog").info(self.rawjsontotextparser(copy.deepcopy(args["data"])),
+                                          extra={"NoStream": True})
+        logging.getLogger("StreamLog").info(self.jsontotextparser(copy.deepcopy(args["data"])),
+                                            extra={"NoFile": True})
 
     def on_package(self, cmd: str, args: dict):
         """For custom package handling in subclasses."""
@@ -876,7 +882,7 @@ def get_base_parser(description: typing.Optional[str] = None):
 def run_as_textclient():
     class TextContext(CommonContext):
         # Text Mode to use !hint and such with games that have no text entry
-        tags = {"AP", "TextOnly"}
+        tags = CommonContext.tags | {"TextOnly"}
         game = ""  # empty matches any game since 0.3.2
         items_handling = 0b111  # receive all items for /received
         want_slot_data = False  # Can't use game specific slot_data

Fill.py

@@ -5,6 +5,8 @@
 from collections import Counter, deque
 
 from BaseClasses import CollectionState, Item, Location, LocationProgressType, MultiWorld
+from Options import Accessibility
+
 from worlds.AutoWorld import call_all
 from worlds.generic.Rules import add_item_rule
 
@@ -70,7 +72,7 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
             spot_to_fill: typing.Optional[Location] = None
 
             # if minimal accessibility, only check whether location is reachable if game not beatable
-            if world.accessibility[item_to_place.player] == 'minimal':
+            if world.worlds[item_to_place.player].options.accessibility == Accessibility.option_minimal:
                 perform_access_check = not world.has_beaten_game(maximum_exploration_state,
                                                                  item_to_place.player) \
                     if single_player_placement else not has_beaten_game
@@ -265,7 +267,7 @@ def fast_fill(world: MultiWorld,
 
 def accessibility_corrections(world: MultiWorld, state: CollectionState, locations, pool=[]):
     maximum_exploration_state = sweep_from_pool(state, pool)
-    minimal_players = {player for player in world.player_ids if world.accessibility[player] == "minimal"}
+    minimal_players = {player for player in world.player_ids if world.worlds[player].options.accessibility == "minimal"}
     unreachable_locations = [location for location in world.get_locations() if location.player in minimal_players and
                              not location.can_reach(maximum_exploration_state)]
     for location in unreachable_locations:
@@ -288,7 +290,7 @@ def inaccessible_location_rules(world: MultiWorld, state: CollectionState, locat
     unreachable_locations = [location for location in locations if not location.can_reach(maximum_exploration_state)]
     if unreachable_locations:
         def forbid_important_item_rule(item: Item):
-            return not ((item.classification & 0b0011) and world.accessibility[item.player] != 'minimal')
+            return not ((item.classification & 0b0011) and world.worlds[item.player].options.accessibility != 'minimal')
 
         for location in unreachable_locations:
             add_item_rule(location, forbid_important_item_rule)
@@ -531,9 +533,9 @@ def balance_multiworld_progression(world: MultiWorld) -> None:
     # If other players are below the threshold value, swap progression in this sphere into earlier spheres,
     #   which gives more locations available by this sphere.
     balanceable_players: typing.Dict[int, float] = {
-        player: world.progression_balancing[player] / 100
+        player: world.worlds[player].options.progression_balancing / 100
         for player in world.player_ids
-        if world.progression_balancing[player] > 0
+        if world.worlds[player].options.progression_balancing > 0
     }
     if not balanceable_players:
         logging.info('Skipping multiworld progression balancing.')
@@ -753,8 +755,6 @@ def failed(warning: str, force: typing.Union[bool, str]) -> None:
         else:  # not reachable with swept state
             non_early_locations[loc.player].append(loc.name)
 
-    # TODO: remove. Preferably by implementing key drop
-    from worlds.alttp.Regions import key_drop_data
     world_name_lookup = world.world_name_lookup
 
     block_value = typing.Union[typing.List[str], typing.Dict[str, typing.Any], str]
@@ -840,14 +840,14 @@ def failed(warning: str, force: typing.Union[bool, str]) -> None:
 
             if "early_locations" in locations:
                 locations.remove("early_locations")
-                for player in worlds:
-                    locations += early_locations[player]
+                for target_player in worlds:
+                    locations += early_locations[target_player]
             if "non_early_locations" in locations:
                 locations.remove("non_early_locations")
-                for player in worlds:
-                    locations += non_early_locations[player]
+                for target_player in worlds:
+                    locations += non_early_locations[target_player]
 
-            block['locations'] = locations
+            block['locations'] = list(dict.fromkeys(locations))
 
             if not block['count']:
                 block['count'] = (min(len(block['items']), len(block['locations'])) if
@@ -897,23 +897,22 @@ def failed(warning: str, force: typing.Union[bool, str]) -> None:
             for item_name in items:
                 item = world.worlds[player].create_item(item_name)
                 for location in reversed(candidates):
-                    if location in key_drop_data:
-                        warn(
-                            f"Can't place '{item_name}' at '{placement.location}', as key drop shuffle locations are not supported yet.")
-                        continue
-                    if not location.item:
-                        if location.item_rule(item):
-                            if location.can_fill(world.state, item, False):
-                                successful_pairs.append((item, location))
-                                candidates.remove(location)
-                                count = count + 1
-                                break
+                    if (location.address is None) == (item.code is None):  # either both None or both not None
+                        if not location.item:
+                            if location.item_rule(item):
+                                if location.can_fill(world.state, item, False):
+                                    successful_pairs.append((item, location))
+                                    candidates.remove(location)
+                                    count = count + 1
+                                    break
+                                else:
+                                    err.append(f"Can't place item at {location} due to fill condition not met.")
                             else:
-                                err.append(f"Can't place item at {location} due to fill condition not met.")
+                                err.append(f"{item_name} not allowed at {location}.")
                         else:
-                            err.append(f"{item_name} not allowed at {location}.")
+                            err.append(f"Cannot place {item_name} into already filled location {location}.")
                     else:
-                        err.append(f"Cannot place {item_name} into already filled location {location}.")
+                        err.append(f"Mismatch between {item_name} and {location}, only one is an event.")
                 if count == maxcount:
                     break
             if count < placement['count']['min']:

Generate.py

@@ -157,7 +157,8 @@ def main(args=None, callback=ERmain):
                         for yaml in weights_cache[path]:
                             if category_name is None:
                                 for category in yaml:
-                                    if category in AutoWorldRegister.world_types and key in Options.common_options:
+                                    if category in AutoWorldRegister.world_types and \
+                                            key in Options.CommonOptions.type_hints:
                                         yaml[category][key] = option
                             elif category_name not in yaml:
                                 logging.warning(f"Meta: Category {category_name} is not present in {path}.")
@@ -340,7 +341,7 @@ def roll_meta_option(option_key, game: str, category_dict: Dict) -> Any:
         return get_choice(option_key, category_dict)
     if game in AutoWorldRegister.world_types:
         game_world = AutoWorldRegister.world_types[game]
-        options = ChainMap(game_world.option_definitions, Options.per_game_common_options)
+        options = game_world.options_dataclass.type_hints
         if option_key in options:
             if options[option_key].supports_weighting:
                 return get_choice(option_key, category_dict)
@@ -445,8 +446,8 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
                                 f"which is not enabled.")
 
     ret = argparse.Namespace()
-    for option_key in Options.per_game_common_options:
-        if option_key in weights and option_key not in Options.common_options:
+    for option_key in Options.PerGameCommonOptions.type_hints:
+        if option_key in weights and option_key not in Options.CommonOptions.type_hints:
             raise Exception(f"Option {option_key} has to be in a game's section, not on its own.")
 
     ret.game = get_choice("game", weights)
@@ -466,16 +467,11 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
         game_weights = weights[ret.game]
 
     ret.name = get_choice('name', weights)
-    for option_key, option in Options.common_options.items():
+    for option_key, option in Options.CommonOptions.type_hints.items():
         setattr(ret, option_key, option.from_any(get_choice(option_key, weights, option.default)))
 
-    for option_key, option in world_type.option_definitions.items():
+    for option_key, option in world_type.options_dataclass.type_hints.items():
         handle_option(ret, game_weights, option_key, option, plando_options)
-    for option_key, option in Options.per_game_common_options.items():
-        # skip setting this option if already set from common_options, defaulting to root option
-        if option_key not in world_type.option_definitions and \
-                (option_key not in Options.common_options or option_key in game_weights):
-            handle_option(ret, game_weights, option_key, option, plando_options)
     if PlandoOptions.items in plando_options:
         ret.plando_items = game_weights.get("plando_items", [])
     if ret.game == "Minecraft" or ret.game == "Ocarina of Time":