Skip to content

Commit

Permalink
perf: getting ape console to launch faster (#2379)
Browse files Browse the repository at this point in the history
  • Loading branch information
@antazoey
antazoey authored Dec 2, 2024
1 parent 92abc86 commit 6b12573
Show file tree
Hide file tree
Showing 12 changed files with 463 additions and 272 deletions.
3 changes: 2 additions & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@
"ijson>=3.1.4,<4",
"ipython>=8.18.1,<9",
"lazyasd>=0.1.4",
"asttokens>=2.4.1,<3", # Peer dependency; w/o pin container build fails.
# Pandas peer-dep: Numpy 2.0 causes issues for some users.
"numpy<2",
"packaging>=23.0,<24",
Expand All @@ -120,7 +121,7 @@
# All version pins dependent on web3[tester]
"eth-abi",
"eth-account",
"eth-typing",
"eth-typing>=3.5.2,<4",
"eth-utils",
"hexbytes",
"py-geth>=5.1.0,<6",
Expand Down
131 changes: 69 additions & 62 deletions src/ape/api/networks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,9 +250,53 @@ def networks(self) -> dict[str, "NetworkAPI"]:
Returns:
dict[str, :class:`~ape.api.networks.NetworkAPI`]
"""
networks = {**self._networks_from_plugins}
return {
**self._networks_from_evmchains,
**self._networks_from_plugins,
**self._custom_networks,
}

@cached_property
def _networks_from_plugins(self) -> dict[str, "NetworkAPI"]:
return {
network_name: network_class(name=network_name, ecosystem=self)
for _, (ecosystem_name, network_name, network_class) in self.plugin_manager.networks
if ecosystem_name == self.name
}

@cached_property
def _networks_from_evmchains(self) -> dict[str, "NetworkAPI"]:
# NOTE: Purposely exclude plugins here so we also prefer plugins.
networks = {
network_name: create_network_type(data["chainId"], data["chainId"])(
name=network_name, ecosystem=self
)
for network_name, data in PUBLIC_CHAIN_META.get(self.name, {}).items()
if network_name not in self._networks_from_plugins
}
forked_networks: dict[str, ForkedNetworkAPI] = {}
for network_name, network in networks.items():
if network_name.endswith("-fork"):
# Already a fork.
continue

fork_network_name = f"{network_name}-fork"
if any(x == fork_network_name for x in networks):
# The forked version of this network is already known.
continue

forked_networks[fork_network_name] = ForkedNetworkAPI(
name=fork_network_name, ecosystem=self
)

return {**networks, **forked_networks}

# Include configured custom networks.
@property
def _custom_networks(self) -> dict[str, "NetworkAPI"]:
"""
Networks from config.
"""
networks: dict[str, "NetworkAPI"] = {}
custom_networks: list[dict] = [
n
for n in self.network_manager.custom_networks
Expand Down Expand Up @@ -300,48 +344,8 @@ def networks(self) -> dict[str, "NetworkAPI"]:
network_api._is_custom = True
networks[net_name] = network_api

# Add any remaining networks from EVM chains here (but don't override).
# NOTE: Only applicable to EVM-based ecosystems, of course.
# Otherwise, this is a no-op.
networks = {**self._networks_from_evmchains, **networks}

return networks

@cached_property
def _networks_from_plugins(self) -> dict[str, "NetworkAPI"]:
return {
network_name: network_class(name=network_name, ecosystem=self)
for _, (ecosystem_name, network_name, network_class) in self.plugin_manager.networks
if ecosystem_name == self.name
}

@cached_property
def _networks_from_evmchains(self) -> dict[str, "NetworkAPI"]:
# NOTE: Purposely exclude plugins here so we also prefer plugins.
networks = {
network_name: create_network_type(data["chainId"], data["chainId"])(
name=network_name, ecosystem=self
)
for network_name, data in PUBLIC_CHAIN_META.get(self.name, {}).items()
if network_name not in self._networks_from_plugins
}
forked_networks: dict[str, ForkedNetworkAPI] = {}
for network_name, network in networks.items():
if network_name.endswith("-fork"):
# Already a fork.
continue

fork_network_name = f"{network_name}-fork"
if any(x == fork_network_name for x in networks):
# The forked version of this network is already known.
continue

forked_networks[fork_network_name] = ForkedNetworkAPI(
name=fork_network_name, ecosystem=self
)

return {**networks, **forked_networks}

def __post_init__(self):
if len(self.networks) == 0:
raise NetworkError("Must define at least one network in ecosystem")
Expand Down Expand Up @@ -1087,12 +1091,13 @@ def providers(self): # -> dict[str, Partial[ProviderAPI]]
Returns:
dict[str, partial[:class:`~ape.api.providers.ProviderAPI`]]
"""
from ape.plugins._utils import clean_plugin_name

providers = {}
for _, plugin_tuple in self._get_plugin_providers():
ecosystem_name, network_name, provider_class = plugin_tuple
provider_name = clean_plugin_name(provider_class.__module__.split(".")[0])
provider_name = (
provider_class.__module__.split(".")[0].replace("_", "-").replace("ape-", "")
)

is_custom_with_config = self._is_custom and self.default_provider_name == provider_name
# NOTE: Custom networks that are NOT from config must work with any provider.
# Also, ensure we are only adding forked providers for forked networks and
Expand All @@ -1101,8 +1106,8 @@ def providers(self): # -> dict[str, Partial[ProviderAPI]]
# TODO: In 0.9, add a better way for class-level ForkedProviders to define
# themselves as "Fork" providers.
if (
self.is_adhoc
or (self.ecosystem.name == ecosystem_name and self.name == network_name)
(self.ecosystem.name == ecosystem_name and self.name == network_name)
or self.is_adhoc
or (
is_custom_with_config
and (
Expand Down Expand Up @@ -1130,6 +1135,11 @@ def _get_plugin_providers(self):
# NOTE: Abstracted for testing purposes.
return self.plugin_manager.providers

def _get_plugin_provider_names(self) -> Iterator[str]:
for _, plugin_tuple in self._get_plugin_providers():
ecosystem_name, network_name, provider_class = plugin_tuple
yield provider_class.__module__.split(".")[0].replace("_", "-").replace("ape-", "")

def get_provider(
self,
provider_name: Optional[str] = None,
Expand All @@ -1147,20 +1157,16 @@ def get_provider(
Returns:
:class:`~ape.api.providers.ProviderAPI`
"""
provider_name = provider_name or self.default_provider_name
if not provider_name:
from ape.managers.config import CONFIG_FILE_NAME

if not (provider_name := provider_name or self.default_provider_name):
raise NetworkError(
f"No default provider for network '{self.name}'. "
f"Set one in your {CONFIG_FILE_NAME}:\n"
"Set one in your pyproject.toml/ape-config.yaml file:\n"
f"\n{self.ecosystem.name}:"
f"\n {self.name}:"
"\n default_provider: <DEFAULT_PROVIDER>"
)

provider_settings = provider_settings or {}

if ":" in provider_name:
# NOTE: Shortcut that allows `--network ecosystem:network:http://...` to work
provider_settings["uri"] = provider_name
Expand All @@ -1170,19 +1176,20 @@ def get_provider(
provider_settings["ipc_path"] = provider_name
provider_name = "node"

# If it can fork Ethereum (and we are asking for it) assume it can fork this one.
# TODO: Refactor this approach to work for custom-forked non-EVM networks.
common_forking_providers = self.network_manager.ethereum.mainnet_fork.providers
if provider_name in self.providers:
provider = self.providers[provider_name](provider_settings=provider_settings)
return _set_provider(provider)

elif self.is_fork and provider_name in common_forking_providers:
provider = common_forking_providers[provider_name](
provider_settings=provider_settings,
network=self,
)
return _set_provider(provider)
elif self.is_fork:
# If it can fork Ethereum (and we are asking for it) assume it can fork this one.
# TODO: Refactor this approach to work for custom-forked non-EVM networks.
common_forking_providers = self.network_manager.ethereum.mainnet_fork.providers
if provider_name in common_forking_providers:
provider = common_forking_providers[provider_name](
provider_settings=provider_settings,
network=self,
)
return _set_provider(provider)

raise ProviderNotFoundError(
provider_name,
Expand Down
79 changes: 18 additions & 61 deletions src/ape/cli/choices.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from typing import TYPE_CHECKING, Any, Optional, Union

import click
from click import BadParameter, Choice, Context, Parameter
from click import Choice, Context, Parameter

from ape.exceptions import (
AccountsError,
Expand Down Expand Up @@ -360,7 +360,7 @@ def __init__(
ecosystem: _NETWORK_FILTER = None,
network: _NETWORK_FILTER = None,
provider: _NETWORK_FILTER = None,
base_type: Optional[type] = None,
base_type: Optional[Union[type, str]] = None,
callback: Optional[Callable] = None,
):
self._base_type = base_type
Expand All @@ -372,15 +372,14 @@ def __init__(
# NOTE: Purposely avoid super().init for performance reasons.

@property
def base_type(self) -> type["ProviderAPI"]:
def base_type(self) -> Union[type["ProviderAPI"], str]:
if self._base_type is not None:
return self._base_type

# perf: property exists to delay import ProviderAPI at init time.
from ape.api.providers import ProviderAPI

self._base_type = ProviderAPI
return ProviderAPI
# perf: Keep base-type as a forward-ref when only using the default.
# so things load faster.
self._base_type = "ProviderAPI"
return self._base_type

@base_type.setter
def base_type(self, value):
Expand All @@ -394,62 +393,20 @@ def get_metavar(self, param):
return "[ecosystem-name][:[network-name][:[provider-name]]]"

def convert(self, value: Any, param: Optional[Parameter], ctx: Optional[Context]) -> Any:
from ape.utils.basemodel import ManagerAccessMixin as access

choice: Optional[Union[str, "ProviderAPI"]]
networks = access.network_manager
if not value:
choice = None

elif value.lower() in ("none", "null"):
choice = _NONE_NETWORK
if not value or value.lower() in ("none", "null"):
return self.callback(ctx, param, _NONE_NETWORK) if self.callback else _NONE_NETWORK

elif self.is_custom_value(value):
# By-pass choice constraints when using custom network.
choice = value
if self.base_type == "ProviderAPI" or isinstance(self.base_type, type):
# Return the provider.
from ape.utils.basemodel import ManagerAccessMixin as access

else:
# Regular conditions.
networks = access.network_manager
try:
# Validate result.
choice = super().convert(value, param, ctx)
except BadParameter:
# Attempt to get the provider anyway.
# Sometimes, depending on the provider, it'll still work.
# (as-is the case for custom-forked networks).
try:
choice = networks.get_provider_from_choice(network_choice=value)

except (EcosystemNotFoundError, NetworkNotFoundError, ProviderNotFoundError) as err:
# This error makes more sense, as it has attempted parsing.
# Show this message as the BadParameter message.
raise click.BadParameter(str(err)) from err

except Exception as err:
# If an error was not raised for some reason, raise a simpler error.
# NOTE: Still avoid showing the massive network options list.
raise click.BadParameter(
"Invalid network choice. Use `ape networks list` to see options."
) from err

if choice not in (None, _NONE_NETWORK) and isinstance(choice, str):
from ape.api.providers import ProviderAPI

if issubclass(self.base_type, ProviderAPI):
# Return the provider.
choice = networks.get_provider_from_choice(network_choice=value)

return self.callback(ctx, param, choice) if self.callback else choice

@classmethod
def is_custom_value(cls, value) -> bool:
return (
value is not None
and isinstance(value, str)
and cls.CUSTOM_NETWORK_PATTERN.match(value) is not None
or str(value).startswith("http://")
or str(value).startswith("https://")
)
value = networks.get_provider_from_choice(network_choice=value)
except (EcosystemNotFoundError, NetworkNotFoundError, ProviderNotFoundError) as err:
self.fail(str(err))

return self.callback(ctx, param, value) if self.callback else value


class OutputFormat(Enum):
Expand Down
17 changes: 15 additions & 2 deletions src/ape/cli/options.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,10 @@ def decorator(f):

def callback(ctx, param, value):
keep_as_choice_str = param.type.base_type is str
provider_obj = _get_provider(value, default, keep_as_choice_str)
try:
provider_obj = _get_provider(value, default, keep_as_choice_str)
except Exception as err:
raise click.BadOptionUsage("--network", str(err), ctx)

if provider_obj:
_update_context_with_network(ctx, provider_obj, requested_network_objects)
Expand Down Expand Up @@ -528,6 +531,10 @@ def handle_parse_result(self, ctx, opts, args):
return IncompatibleOption


def _project_path_callback(ctx, param, val):
return Path(val) if val else Path.cwd()


def _project_callback(ctx, param, val):
if "--help" in sys.argv or "-h" in sys.argv:
# Perf: project option is eager; have to check sys.argv to
Expand Down Expand Up @@ -560,10 +567,16 @@ def _project_callback(ctx, param, val):


def project_option(**kwargs):
_type = kwargs.pop("type", None)
callback = (
_project_path_callback
if (isinstance(_type, type) and issubclass(_type, Path))
else _project_callback
)
return click.option(
"--project",
help="The path to a local project or manifest",
callback=_project_callback,
callback=callback,
metavar="PATH",
is_eager=True,
**kwargs,
Expand Down
Loading

0 comments on commit 6b12573

Please sign in to comment.