diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index e0aed6a973..afc59002f4 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [2.7, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 5825fd8124..34d261d4f4 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-16.04, windows-latest, macos-latest] - python-version: [2.7, 3.7] + python-version: [3.7] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.isort.cfg b/.isort.cfg index de9bf40e83..7d21b11778 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=88 -known_third_party=OpenSSL, SCons, autobahn, jsonrpc, twisted, zope +known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope diff --git a/.pylintrc b/.pylintrc index e21dfef9f5..bb68f8a095 100644 --- a/.pylintrc +++ b/.pylintrc @@ -14,7 +14,6 @@ disable= too-few-public-methods, useless-object-inheritance, useless-import-alias, - fixme, bad-option-value, ; PY2 Compat diff --git a/HISTORY.rst b/HISTORY.rst index 9e5bff4e7d..5687765d7f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,15 +8,54 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.1.0 (2021-01-28) +~~~~~~~~~~~~~~~~~~ + +* **PlatformIO Home** + + - Boosted PlatformIO Home performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack + - Added a new ``--session-id`` option to `pio home `__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 `_) + +* **Build System** + + - Upgraded build engine to the SCons 4.1 (`release notes `_) + - Refactored a workaround for a maximum command line character limitation (`issue #3792 `_) + - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) + +* **Package Management** + + - New options for `pio system prune `__ command: + + + ``--dry-run`` option to show data that will be removed + + ``--core-packages`` option to remove unnecessary core packages + + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 `_) + + - Added new `check_prune_system_threshold `__ setting + - Disabled automatic removal of unnecessary development platform packages (`issue #3708 `_, `issue #3770 `_) + - Fixed an issue when unnecessary packages were removed in ``update --dry-run`` mode (`issue #3809 `_) + - Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source (`issue #3816 `_) + +* **Debugging** + + - Configure a custom debug adapter speed using a new `debug_speed `__ option (`issue #3799 `_) + - Handle debugging server's "ready_pattern" in "stderr" output + +* **Miscellaneous** + + - Improved listing of `multicast DNS services `_ + - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 `_) + - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 `_) + - Fixed an issue when `pio device monitor --eol `__ and "send_on_enter" filter do not work properly (`issue #3787 `_) + 5.0.4 (2020-12-30) ~~~~~~~~~~~~~~~~~~ - Added "Core" suffix when showing PlatformIO Core version using ``pio --version`` command -- Improved ``.ccls`` configuration file for Emacs, Vim, and Sublime Text integrations +- Improved ".ccls" configuration file for Emacs, Vim, and Sublime Text integrations - Updated analysis tools: - * `Cppcheck `__ v2.3 with improved C++ parser and several new MISRA rules - * `PVS-Studio `__ v7.11 with new diagnostics and updated mass suppression mechanism + * `Cppcheck `__ v2.3 with improved C++ parser and several new MISRA rules + * `PVS-Studio `__ v7.11 with new diagnostics and updated mass suppression mechanism - Show a warning message about deprecated support for Python 2 and Python 3.5 - Do not provide "intelliSenseMode" option when generating configuration for VSCode C/C++ extension diff --git a/Makefile b/Makefile index 3ddd1e27da..a73d9cba4b 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ lint: pylint -j 6 --rcfile=./.pylintrc ./tests isort: - isort -rc ./platformio - isort -rc ./tests + isort ./platformio + isort ./tests format: black --target-version py27 ./platformio diff --git a/docs b/docs index 9db46dccef..25edd66d55 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9db46dccef0a765770e71b64bf29b6e1d91df403 +Subproject commit 25edd66d5514cc5a0c8d5a01f4752de3e98a03d0 diff --git a/examples b/examples index 161ae7302b..8a6e639b2b 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 161ae7302b3508cadb10f5552de9f996731976ac +Subproject commit 8a6e639b2bcb18dec63bc010f359b49f85084b45 diff --git a/platformio/__init__.py b/platformio/__init__.py index a540b3ffac..5ed64a4ea2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, 4) +VERSION = (5, 1, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -47,10 +47,10 @@ __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.3.1", + "contrib-piohome": "~3.3.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.2", "tool-cppcheck": "~1.230.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.11.0", diff --git a/platformio/app.py b/platformio/app.py index 9b22b638cf..04d02c39c0 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -55,6 +55,10 @@ def projects_dir_validate(projects_dir): "description": "Check for the platform updates interval (days)", "value": 7, }, + "check_prune_system_threshold": { + "description": "Check for pruning unnecessary data threshold (megabytes)", + "value": 1024, + }, "enable_cache": { "description": "Enable caching for HTTP API requests", "value": True, diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b1e8ffbd64..6a060dd134 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -81,12 +81,19 @@ IDE_EXTRA_DATA={}, ) +# Declare command verbose messages +command_strings = dict( + ARCOM="Archiving", + LINKCOM="Linking", + RANLIBCOM="Indexing", + ASCOM="Compiling", + ASPPCOM="Compiling", + CCCOM="Compiling", + CXXCOM="Compiling", +) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): - DEFAULT_ENV_OPTIONS["ARCOMSTR"] = "Archiving $TARGET" - DEFAULT_ENV_OPTIONS["LINKCOMSTR"] = "Linking $TARGET" - DEFAULT_ENV_OPTIONS["RANLIBCOMSTR"] = "Indexing $TARGET" - for k in ("ASCOMSTR", "ASPPCOMSTR", "CCCOMSTR", "CXXCOMSTR"): - DEFAULT_ENV_OPTIONS[k] = "Compiling $TARGET" + for name, value in command_strings.items(): + DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value) env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 150a832e96..90b6517a8a 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -41,7 +41,7 @@ # should hold the compilation database, otherwise, the file defaults to compile_commands.json, # which is the name that most clang tools search for by default. -# TODO: Is there a better way to do this than this global? Right now this exists so that the +# Is there a better way to do this than this global? Right now this exists so that the # emitter we add can record all of the things it emits, so that the scanner for the top level # compilation database can access the complete list, and also so that the writer has easy # access to write all of the files. But it seems clunky. How can the emitter and the scanner @@ -104,7 +104,7 @@ def EmitCompilationDbEntry(target, source, env): __COMPILATIONDB_ENV=env, ) - # TODO: Technically, these next two lines should not be required: it should be fine to + # Technically, these next two lines should not be required: it should be fine to # cache the entries. However, they don't seem to update properly. Since they are quick # to re-generate disable caching and sidestep this problem. env.AlwaysBuild(entry) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 8248c06aaa..de7e0cc8be 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -17,7 +17,8 @@ import os from glob import glob -from SCons.Defaults import processDefines # pylint: disable=import-error +import SCons.Defaults # pylint: disable=import-error +import SCons.Subst # pylint: disable=import-error from platformio.compat import glob_escape from platformio.package.manager.core import get_core_package_dir @@ -58,8 +59,16 @@ def _dump_includes(env): for g in toolchain_incglobs: includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)]) + # include Unity framework if there are tests in project includes["unity"] = [] - unity_dir = get_core_package_dir("tool-unity") + auto_install_unity = False + test_dir = env.GetProjectConfig().get_optional_dir("test") + if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]: + auto_install_unity = True + unity_dir = get_core_package_dir( + "tool-unity", + auto_install=auto_install_unity, + ) if unity_dir: includes["unity"].append(unity_dir) @@ -92,7 +101,7 @@ def _get_gcc_defines(env): def _dump_defines(env): defines = [] # global symbols - for item in processDefines(env.get("CPPDEFINES", [])): + for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])): item = item.strip() if item: defines.append(env.subst(item).replace("\\", "")) @@ -141,25 +150,17 @@ def _get_svd_path(env): return None -def _escape_build_flag(flags): - return [flag if " " not in flag else '"%s"' % flag for flag in flags] +def _subst_cmd(env, cmd): + args = env.subst_list(cmd, SCons.Subst.SUBST_CMD)[0] + return " ".join([SCons.Subst.quote_spaces(arg) for arg in args]) def DumpIDEData(env, globalenv): """ env here is `projenv`""" - env["__escape_build_flag"] = _escape_build_flag - - LINTCCOM = ( - "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" - ) - LINTCXXCOM = ( - "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" - ) - data = { "env_name": env["PIOENV"], - "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], + "libsource_dirs": [env.subst(item) for item in env.GetLibSourceDirs()], "defines": _dump_defines(env), "includes": _dump_includes(env), "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), @@ -181,7 +182,7 @@ def DumpIDEData(env, globalenv): env_ = env.Clone() # https://github.com/platformio/platformio-atom-ide/issues/34 _new_defines = [] - for item in processDefines(env_.get("CPPDEFINES", [])): + for item in SCons.Defaults.processDefines(env_.get("CPPDEFINES", [])): item = item.replace('\\"', '"') if " " in item: _new_defines.append(item.replace(" ", "\\\\ ")) @@ -189,7 +190,13 @@ def DumpIDEData(env, globalenv): _new_defines.append(item) env_.Replace(CPPDEFINES=_new_defines) - data.update({"cc_flags": env_.subst(LINTCCOM), "cxx_flags": env_.subst(LINTCXXCOM)}) + # export C/C++ build flags + data.update( + { + "cc_flags": _subst_cmd(env_, "$CFLAGS $CCFLAGS $CPPFLAGS"), + "cxx_flags": _subst_cmd(env_, "$CXXFLAGS $CCFLAGS $CPPFLAGS"), + } + ) return data diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 04f1304f1f..d386bd1448 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -14,15 +14,30 @@ from __future__ import absolute_import -from hashlib import md5 -from os import makedirs -from os.path import isdir, isfile, join +import hashlib +import os +import re + +from SCons.Platform import TempFileMunge # pylint: disable=import-error +from SCons.Subst import quote_spaces # pylint: disable=import-error from platformio.compat import WINDOWS, hashlib_encode_data -# Windows CLI has limit with command length to 8192 -# Leave 2000 chars for flags and other options -MAX_LINE_LENGTH = 6000 if WINDOWS else 128072 +# There are the next limits depending on a platform: +# - Windows = 8192 +# - Unix = 131072 +# We need ~256 characters for a temporary file path +MAX_LINE_LENGTH = (8192 if WINDOWS else 131072) - 256 + +WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)") + + +def tempfile_arg_esc_func(arg): + arg = quote_spaces(arg) + if not WINDOWS: + return arg + # GCC requires double Windows slashes, let's use UNIX separator + return WINPATHSEP_RE.sub(r"/\1", arg) def long_sources_hook(env, sources): @@ -41,30 +56,14 @@ def long_sources_hook(env, sources): return '@"%s"' % _file_long_data(env, " ".join(data)) -def long_incflags_hook(env, incflags): - _incflags = env.subst(incflags).replace("\\", "/") - if len(_incflags) < MAX_LINE_LENGTH: - return incflags - - # fix space in paths - data = [] - for line in _incflags.split(" -I"): - line = line.strip() - if not line.startswith("-I"): - line = "-I" + line - data.append('-I"%s"' % line[2:]) - - return '@"%s"' % _file_long_data(env, " ".join(data)) - - def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") - if not isdir(build_dir): - makedirs(build_dir) - tmp_file = join( - build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest() + if not os.path.isdir(build_dir): + os.makedirs(build_dir) + tmp_file = os.path.join( + build_dir, "longcmd-%s" % hashlib.md5(hashlib_encode_data(data)).hexdigest() ) - if isfile(tmp_file): + if os.path.isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: fp.write(data) @@ -76,17 +75,21 @@ def exists(_): def generate(env): - env.Replace(_long_sources_hook=long_sources_hook) - env.Replace(_long_incflags_hook=long_incflags_hook) - coms = {} - for key in ("ARCOM", "LINKCOM"): - coms[key] = env.get(key, "").replace( - "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" - ) - for key in ("_CCCOMCOM", "ASPPCOM"): - coms[key] = env.get(key, "").replace( - "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}" - ) - env.Replace(**coms) + kwargs = dict( + _long_sources_hook=long_sources_hook, + TEMPFILE=TempFileMunge, + MAXLINELENGTH=MAX_LINE_LENGTH, + TEMPFILEARGESCFUNC=tempfile_arg_esc_func, + TEMPFILESUFFIX=".tmp", + TEMPFILEDIR="$BUILD_DIR", + ) + + for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"): + kwargs[name] = "${TEMPFILE('%s','$%sSTR')}" % (env.get(name), name) + + kwargs["ARCOM"] = env.get("ARCOM", "").replace( + "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" + ) + env.Replace(**kwargs) return env diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 62ce4f7394..7561c33840 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -176,6 +176,7 @@ def _cleanup_cmds(items): tool_name, tool_settings, ), + speed=env_options.get("debug_speed", tool_settings.get("speed")), server=server_options, ) return result diff --git a/platformio/commands/debug/process/server.py b/platformio/commands/debug/process/server.py index 7bd5d4850b..7a302c9be6 100644 --- a/platformio/commands/debug/process/server.py +++ b/platformio/commands/debug/process/server.py @@ -124,16 +124,25 @@ def spawn(self, patterns): # pylint: disable=too-many-branches @defer.inlineCallbacks def _wait_until_ready(self): - timeout = 10 + ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") + timeout = 60 if ready_pattern else 10 elapsed = 0 delay = 0.5 auto_ready_delay = 0.5 while not self._ready and not self._process_ended and elapsed < timeout: yield self.async_sleep(delay) - if not self.debug_options.get("server", {}).get("ready_pattern"): + if not ready_pattern: self._ready = self._last_activity < (time.time() - auto_ready_delay) elapsed += delay + def _check_ready_by_pattern(self, data): + if self._ready: + return self._ready + ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") + if ready_pattern: + self._ready = ready_pattern.encode() in data + return self._ready + @staticmethod def async_sleep(secs): d = defer.Deferred() @@ -147,11 +156,11 @@ def outReceived(self, data): super(DebugServer, self).outReceived( escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data ) - if self._ready: - return - ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") - if ready_pattern: - self._ready = ready_pattern.encode() in data + self._check_ready_by_pattern(data) + + def errReceived(self, data): + super(DebugServer, self).errReceived(data) + self._check_ready_by_pattern(data) def processEnded(self, reason): self._process_ended = True diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index 2e254742c7..fd385a46fa 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -179,7 +179,9 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches for name in os.listdir(filters_dir): if not name.endswith(".py"): continue - device_helpers.load_monitor_filter(os.path.join(filters_dir, name)) + device_helpers.load_monitor_filter( + os.path.join(filters_dir, name), options=kwargs + ) project_options = {} try: @@ -193,9 +195,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches if "platform" in project_options: with fs.cd(kwargs["project_dir"]): platform = PlatformFactory.new(project_options["platform"]) - device_helpers.register_platform_filters( - platform, kwargs["project_dir"], kwargs["environment"] - ) + device_helpers.register_platform_filters(platform, options=kwargs) if not kwargs["port"]: ports = util.get_serial_ports(filter_hwid=True) diff --git a/platformio/commands/device/filters/base.py b/platformio/commands/device/filters/base.py index 5c6d0400f1..6745a626f3 100644 --- a/platformio/commands/device/filters/base.py +++ b/platformio/commands/device/filters/base.py @@ -18,12 +18,13 @@ class DeviceMonitorFilter(miniterm.Transform): - def __init__(self, project_dir=None, environment=None): + def __init__(self, options=None): """ Called by PlatformIO to pass context """ miniterm.Transform.__init__(self) - self.project_dir = project_dir - self.environment = environment + self.options = options or {} + self.project_dir = self.options.get("project_dir") + self.environment = self.options.get("environment") self.config = ProjectConfig.get_instance() if not self.environment: diff --git a/platformio/commands/device/filters/send_on_enter.py b/platformio/commands/device/filters/send_on_enter.py index 10ca21031b..8300c98040 100644 --- a/platformio/commands/device/filters/send_on_enter.py +++ b/platformio/commands/device/filters/send_on_enter.py @@ -22,10 +22,17 @@ def __init__(self, *args, **kwargs): super(SendOnEnter, self).__init__(*args, **kwargs) self._buffer = "" + if self.options.get("eol") == "CR": + self._eol = "\r" + elif self.options.get("eol") == "LF": + self._eol = "\n" + else: + self._eol = "\r\n" + def tx(self, text): self._buffer += text - if self._buffer.endswith("\r\n"): - text = self._buffer[:-2] + if self._buffer.endswith(self._eol): + text = self._buffer[: len(self._eol) * -1] self._buffer = "" return text return "" diff --git a/platformio/commands/device/helpers.py b/platformio/commands/device/helpers.py index 3bfe8fc693..a65b4895db 100644 --- a/platformio/commands/device/helpers.py +++ b/platformio/commands/device/helpers.py @@ -76,7 +76,7 @@ def get_board_hwids(project_dir, platform, board): return platform.board_config(board).get("build.hwids", []) -def load_monitor_filter(path, project_dir=None, environment=None): +def load_monitor_filter(path, options=None): name = os.path.basename(path) name = name[: name.find(".")] module = load_python_module("platformio.commands.device.filters.%s" % name, path) @@ -87,12 +87,12 @@ def load_monitor_filter(path, project_dir=None, environment=None): or cls == DeviceMonitorFilter ): continue - obj = cls(project_dir, environment) + obj = cls(options) miniterm.TRANSFORMATIONS[obj.NAME] = obj return True -def register_platform_filters(platform, project_dir, environment): +def register_platform_filters(platform, options=None): monitor_dir = os.path.join(platform.get_dir(), "monitor") if not os.path.isdir(monitor_dir): return @@ -103,4 +103,4 @@ def register_platform_filters(platform, project_dir, environment): path = os.path.join(monitor_dir, name) if not os.path.isfile(path): continue - load_monitor_filter(path, project_dir, environment) + load_monitor_filter(path, options) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 6cb26ed955..2973bdd25b 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -12,20 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-locals,too-many-statements - import mimetypes -import socket -from os.path import isdir import click -from platformio import exception -from platformio.compat import WINDOWS -from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite +from platformio.commands.home.helpers import is_port_used +from platformio.compat import ensure_python3 -@click.command("home", short_help="UI to manage PlatformIO") +@click.command("home", short_help="GUI to manage PlatformIO") @click.option("--port", type=int, default=8008, help="HTTP port, default=8008") @click.option( "--host", @@ -45,61 +40,30 @@ "are connected. Default is 0 which means never auto shutdown" ), ) -def cli(port, host, no_open, shutdown_timeout): - # pylint: disable=import-error, import-outside-toplevel - - # import contrib modules - inject_contrib_pysite() - - from autobahn.twisted.resource import WebSocketResource - from twisted.internet import reactor - from twisted.web import server - from twisted.internet.error import CannotListenError - - from platformio.commands.home.rpc.handlers.app import AppRPC - from platformio.commands.home.rpc.handlers.ide import IDERPC - from platformio.commands.home.rpc.handlers.misc import MiscRPC - from platformio.commands.home.rpc.handlers.os import OSRPC - from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC - from platformio.commands.home.rpc.handlers.project import ProjectRPC - from platformio.commands.home.rpc.handlers.account import AccountRPC - from platformio.commands.home.rpc.server import JSONRPCServerFactory - from platformio.commands.home.web import WebRoot - - factory = JSONRPCServerFactory(shutdown_timeout) - factory.addHandler(AppRPC(), namespace="app") - factory.addHandler(IDERPC(), namespace="ide") - factory.addHandler(MiscRPC(), namespace="misc") - factory.addHandler(OSRPC(), namespace="os") - factory.addHandler(PIOCoreRPC(), namespace="core") - factory.addHandler(ProjectRPC(), namespace="project") - factory.addHandler(AccountRPC(), namespace="account") - - contrib_dir = get_core_package_dir("contrib-piohome") - if not isdir(contrib_dir): - raise exception.PlatformioException("Invalid path to PIO Home Contrib") +@click.option( + "--session-id", + help=( + "A unique session identifier to keep PIO Home isolated from other instances " + "and protect from 3rd party access" + ), +) +def cli(port, host, no_open, shutdown_timeout, session_id): + ensure_python3() # Ensure PIO Home mimetypes are known mimetypes.add_type("text/html", ".html") mimetypes.add_type("text/css", ".css") mimetypes.add_type("application/javascript", ".js") - root = WebRoot(contrib_dir) - root.putChild(b"wsrpc", WebSocketResource(factory)) - site = server.Site(root) - # hook for `platformio-node-helpers` if host == "__do_not_start__": return - already_started = is_port_used(host, port) - home_url = "http://%s:%d" % (host, port) - if not no_open: - if already_started: - click.launch(home_url) - else: - reactor.callLater(1, lambda: click.launch(home_url)) - + home_url = "http://%s:%d%s" % ( + host, + port, + ("/session/%s/" % session_id) if session_id else "/", + ) click.echo( "\n".join( [ @@ -108,45 +72,28 @@ def cli(port, host, no_open, shutdown_timeout): " /\\-_--\\ PlatformIO Home", "/ \\_-__\\", "|[]| [] | %s" % home_url, - "|__|____|______________%s" % ("_" * len(host)), + "|__|____|__%s" % ("_" * len(home_url)), ] ) ) click.echo("") click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url) - try: - reactor.listenTCP(port, site, interface=host) - except CannotListenError as e: - click.secho(str(e), fg="red", err=True) - already_started = True - - if already_started: + if is_port_used(host, port): click.secho( "PlatformIO Home server is already started in another process.", fg="yellow" ) + if not no_open: + click.launch(home_url) return - click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") - - reactor.run() - + # pylint: disable=import-outside-toplevel + from platformio.commands.home.run import run_server -def is_port_used(host, port): - socket.setdefaulttimeout(1) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if WINDOWS: - try: - s.bind((host, port)) - s.close() - return False - except (OSError, socket.error): - pass - else: - try: - s.connect((host, port)) - s.close() - except socket.error: - return False - - return True + run_server( + host=host, + port=port, + no_open=no_open, + shutdown_timeout=shutdown_timeout, + home_url=home_url, + ) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index aff9228126..5c6e0c888f 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -12,36 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=keyword-arg-before-vararg,arguments-differ,signature-differs +import socket import requests -from twisted.internet import defer # pylint: disable=import-error -from twisted.internet import reactor # pylint: disable=import-error -from twisted.internet import threads # pylint: disable=import-error +from starlette.concurrency import run_in_threadpool from platformio import util +from platformio.compat import WINDOWS from platformio.proc import where_is_program class AsyncSession(requests.Session): - def __init__(self, n=None, *args, **kwargs): - if n: - pool = reactor.getThreadPool() - pool.adjustPoolsize(0, n) - - super(AsyncSession, self).__init__(*args, **kwargs) - - def request(self, *args, **kwargs): + async def request( # pylint: disable=signature-differs,invalid-overridden-method + self, *args, **kwargs + ): func = super(AsyncSession, self).request - return threads.deferToThread(func, *args, **kwargs) - - def wrap(self, *args, **kwargs): # pylint: disable=no-self-use - return defer.ensureDeferred(*args, **kwargs) + return await run_in_threadpool(func, *args, **kwargs) @util.memoized(expire="60s") def requests_session(): - return AsyncSession(n=5) + return AsyncSession() @util.memoized(expire="60s") @@ -49,3 +40,23 @@ def get_core_fullpath(): return where_is_program( "platformio" + (".exe" if "windows" in util.get_systype() else "") ) + + +def is_port_used(host, port): + socket.setdefaulttimeout(1) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if WINDOWS: + try: + s.bind((host, port)) + s.close() + return False + except (OSError, socket.error): + pass + else: + try: + s.connect((host, port)) + s.close() + except socket.error: + return False + + return True diff --git a/platformio/commands/home/rpc/handlers/account.py b/platformio/commands/home/rpc/handlers/account.py index d28379f83f..337d780aaf 100644 --- a/platformio/commands/home/rpc/handlers/account.py +++ b/platformio/commands/home/rpc/handlers/account.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import jsonrpc # pylint: disable=import-error +import jsonrpc from platformio.clients.account import AccountClient -class AccountRPC(object): +class AccountRPC: @staticmethod def call_client(method, *args, **kwargs): try: diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 1fd49e229c..3c0ce4657b 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -20,7 +20,7 @@ from platformio.project.helpers import get_project_core_dir, is_platformio_project -class AppRPC(object): +class AppRPC: APPSTATE_PATH = join(get_project_core_dir(), "homestate.json") diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index e3ad75f32c..ed95b738ab 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -14,11 +14,12 @@ import time -import jsonrpc # pylint: disable=import-error -from twisted.internet import defer # pylint: disable=import-error +import jsonrpc +from platformio.compat import get_running_loop -class IDERPC(object): + +class IDERPC: def __init__(self): self._queue = {} @@ -28,14 +29,14 @@ def send_command(self, sid, command, params): code=4005, message="PIO Home IDE agent is not started" ) while self._queue[sid]: - self._queue[sid].pop().callback( + self._queue[sid].pop().set_result( {"id": time.time(), "method": command, "params": params} ) def listen_commands(self, sid=0): if sid not in self._queue: self._queue[sid] = [] - self._queue[sid].append(defer.Deferred()) + self._queue[sid].append(get_running_loop().create_future()) return self._queue[sid][-1] def open_project(self, sid, project_dir): diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index a4bdc6522a..c16a6cc9db 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -15,14 +15,13 @@ import json import time -from twisted.internet import defer, reactor # pylint: disable=import-error - from platformio.cache import ContentCache from platformio.commands.home.rpc.handlers.os import OSRPC +from platformio.compat import create_task -class MiscRPC(object): - def load_latest_tweets(self, data_url): +class MiscRPC: + async def load_latest_tweets(self, data_url): cache_key = ContentCache.key_from_args(data_url, "tweets") cache_valid = "180d" with ContentCache() as cc: @@ -31,22 +30,20 @@ def load_latest_tweets(self, data_url): cache_data = json.loads(cache_data) # automatically update cache in background every 12 hours if cache_data["time"] < (time.time() - (3600 * 12)): - reactor.callLater( - 5, self._preload_latest_tweets, data_url, cache_key, cache_valid + create_task( + self._preload_latest_tweets(data_url, cache_key, cache_valid) ) return cache_data["result"] - result = self._preload_latest_tweets(data_url, cache_key, cache_valid) - return result + return await self._preload_latest_tweets(data_url, cache_key, cache_valid) @staticmethod - @defer.inlineCallbacks - def _preload_latest_tweets(data_url, cache_key, cache_valid): - result = json.loads((yield OSRPC.fetch_content(data_url))) + async def _preload_latest_tweets(data_url, cache_key, cache_valid): + result = json.loads((await OSRPC.fetch_content(data_url))) with ContentCache() as cc: cc.set( cache_key, json.dumps({"time": int(time.time()), "result": result}), cache_valid, ) - defer.returnValue(result) + return result diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 448c633a50..f104297830 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,25 +14,23 @@ from __future__ import absolute_import +import glob import io import os import shutil from functools import cmp_to_key import click -from twisted.internet import defer # pylint: disable=import-error from platformio import __default_requests_timeout__, fs, util from platformio.cache import ContentCache from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding, glob_recursive -class OSRPC(object): +class OSRPC: @staticmethod - @defer.inlineCallbacks - def fetch_content(uri, data=None, headers=None, cache_valid=None): + async def fetch_content(uri, data=None, headers=None, cache_valid=None): if not headers: headers = { "User-Agent": ( @@ -46,18 +44,18 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): if cache_key: result = cc.get(cache_key) if result is not None: - defer.returnValue(result) + return result # check internet before and resolve issue with 60 seconds timeout ensure_internet_on(raise_exception=True) session = helpers.requests_session() if data: - r = yield session.post( + r = await session.post( uri, data=data, headers=headers, timeout=__default_requests_timeout__ ) else: - r = yield session.get( + r = await session.get( uri, headers=headers, timeout=__default_requests_timeout__ ) @@ -66,11 +64,11 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): if cache_valid: with ContentCache() as cc: cc.set(cache_key, result, cache_valid) - defer.returnValue(result) + return result - def request_content(self, uri, data=None, headers=None, cache_valid=None): + async def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith("http"): - return self.fetch_content(uri, data, headers, cache_valid) + return await self.fetch_content(uri, data, headers, cache_valid) if os.path.isfile(uri): with io.open(uri, encoding="utf-8") as fp: return fp.read() @@ -82,13 +80,11 @@ def open_url(url): @staticmethod def reveal_file(path): - return click.launch( - path.encode(get_filesystem_encoding()) if PY2 else path, locate=True - ) + return click.launch(path, locate=True) @staticmethod def open_file(path): - return click.launch(path.encode(get_filesystem_encoding()) if PY2 else path) + return click.launch(path) @staticmethod def is_file(path): @@ -121,7 +117,9 @@ def glob(pathnames, root=None): result = set() for pathname in pathnames: result |= set( - glob_recursive(os.path.join(root, pathname) if root else pathname) + glob.glob( + os.path.join(root, pathname) if root else pathname, recursive=True + ) ) return list(result) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 7a16f9c64a..d74095abaf 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,23 +17,15 @@ import json import os import sys -from io import BytesIO, StringIO +from io import StringIO import click -import jsonrpc # pylint: disable=import-error -from twisted.internet import defer # pylint: disable=import-error -from twisted.internet import threads # pylint: disable=import-error -from twisted.internet import utils # pylint: disable=import-error +import jsonrpc +from starlette.concurrency import run_in_threadpool -from platformio import __main__, __version__, fs +from platformio import __main__, __version__, fs, proc from platformio.commands.home import helpers -from platformio.compat import ( - PY2, - get_filesystem_encoding, - get_locale_encoding, - is_bytes, - string_types, -) +from platformio.compat import get_locale_encoding, is_bytes try: from thread import get_ident as thread_get_ident @@ -52,13 +44,11 @@ def __getattr__(self, name): def _ensure_thread_buffer(self, thread_id): if thread_id not in self._buffers: - self._buffers[thread_id] = BytesIO() if PY2 else StringIO() + self._buffers[thread_id] = StringIO() def write(self, value): thread_id = thread_get_ident() self._ensure_thread_buffer(thread_id) - if PY2 and isinstance(value, unicode): # pylint: disable=undefined-variable - value = value.encode() return self._buffers[thread_id].write( value.decode() if is_bytes(value) else value ) @@ -74,7 +64,7 @@ def get_value_and_reset(self): return result -class PIOCoreRPC(object): +class PIOCoreRPC: @staticmethod def version(): return __version__ @@ -89,16 +79,9 @@ def setup_multithreading_std_streams(): sys.stderr = PIOCoreRPC.thread_stderr @staticmethod - def call(args, options=None): - return defer.maybeDeferred(PIOCoreRPC._call_generator, args, options) - - @staticmethod - @defer.inlineCallbacks - def _call_generator(args, options=None): + async def call(args, options=None): for i, arg in enumerate(args): - if isinstance(arg, string_types): - args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg - else: + if not isinstance(arg, str): args[i] = str(arg) options = options or {} @@ -106,27 +89,34 @@ def _call_generator(args, options=None): try: if options.get("force_subprocess"): - result = yield PIOCoreRPC._call_subprocess(args, options) - defer.returnValue(PIOCoreRPC._process_result(result, to_json)) - else: - result = yield PIOCoreRPC._call_inline(args, options) - try: - defer.returnValue(PIOCoreRPC._process_result(result, to_json)) - except ValueError: - # fall-back to subprocess method - result = yield PIOCoreRPC._call_subprocess(args, options) - defer.returnValue(PIOCoreRPC._process_result(result, to_json)) + result = await PIOCoreRPC._call_subprocess(args, options) + return PIOCoreRPC._process_result(result, to_json) + result = await PIOCoreRPC._call_inline(args, options) + try: + return PIOCoreRPC._process_result(result, to_json) + except ValueError: + # fall-back to subprocess method + result = await PIOCoreRPC._call_subprocess(args, options) + return PIOCoreRPC._process_result(result, to_json) except Exception as e: # pylint: disable=bare-except raise jsonrpc.exceptions.JSONRPCDispatchException( code=4003, message="PIO Core Call Error", data=str(e) ) @staticmethod - def _call_inline(args, options): + async def _call_subprocess(args, options): + result = await run_in_threadpool( + proc.exec_command, + [helpers.get_core_fullpath()] + args, + cwd=options.get("cwd") or os.getcwd(), + ) + return (result["out"], result["err"], result["returncode"]) + + @staticmethod + async def _call_inline(args, options): PIOCoreRPC.setup_multithreading_std_streams() - cwd = options.get("cwd") or os.getcwd() - def _thread_task(): + def _thread_safe_call(args, cwd): with fs.cd(cwd): exit_code = __main__.main(["-c"] + args) return ( @@ -135,16 +125,8 @@ def _thread_task(): exit_code, ) - return threads.deferToThread(_thread_task) - - @staticmethod - def _call_subprocess(args, options): - cwd = (options or {}).get("cwd") or os.getcwd() - return utils.getProcessOutputAndValue( - helpers.get_core_fullpath(), - args, - path=cwd, - env={k: v for k, v in os.environ.items() if "%" not in k}, + return await run_in_threadpool( + _thread_safe_call, args=args, cwd=options.get("cwd") or os.getcwd() ) @staticmethod diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index eb9cd23721..c8ec6f5431 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -18,12 +18,11 @@ import shutil import time -import jsonrpc # pylint: disable=import-error +import jsonrpc from platformio import exception, fs from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC -from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig @@ -32,7 +31,7 @@ from platformio.project.options import get_config_options_schema -class ProjectRPC(object): +class ProjectRPC: @staticmethod def config_call(init_kwargs, method, *args): assert isinstance(init_kwargs, dict) @@ -185,7 +184,7 @@ def get_project_examples(): ) return sorted(result, key=lambda data: data["platform"]["title"]) - def init(self, board, framework, project_dir): + async def init(self, board, framework, project_dir): assert project_dir state = AppRPC.load_state() if not os.path.isdir(project_dir): @@ -198,14 +197,13 @@ def init(self, board, framework, project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call( + await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) - d.addCallback(self._generate_project_main, project_dir, framework) - return d + return self._generate_project_main(project_dir, framework) @staticmethod - def _generate_project_main(_, project_dir, framework): + def _generate_project_main(project_dir, framework): main_content = None if framework == "arduino": main_content = "\n".join( @@ -252,10 +250,8 @@ def _generate_project_main(_, project_dir, framework): fp.write(main_content.strip()) return project_dir - def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + async def import_arduino(self, board, use_arduino_libs, arduino_project_dir): board = str(board) - if arduino_project_dir and PY2: - arduino_project_dir = arduino_project_dir.encode(get_filesystem_encoding()) # don't import PIO Project if is_platformio_project(arduino_project_dir): return arduino_project_dir @@ -293,14 +289,9 @@ def import_arduino(self, board, use_arduino_libs, arduino_project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call( + await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) - d.addCallback(self._finalize_arduino_import, project_dir, arduino_project_dir) - return d - - @staticmethod - def _finalize_arduino_import(_, project_dir, arduino_project_dir): with fs.cd(project_dir): config = ProjectConfig() src_dir = config.get_optional_dir("src") @@ -310,7 +301,7 @@ def _finalize_arduino_import(_, project_dir, arduino_project_dir): return project_dir @staticmethod - def import_pio(project_dir): + async def import_pio(project_dir): if not project_dir or not is_platformio_project(project_dir): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir @@ -328,8 +319,7 @@ def import_pio(project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call( + await PIOCoreRPC.call( args, options={"cwd": new_project_dir, "force_subprocess": True} ) - d.addCallback(lambda _: new_project_dir) - return d + return new_project_dir diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index 1924754f38..8ba41da9b1 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -12,90 +12,107 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=import-error +import inspect +import json import click import jsonrpc -from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol -from jsonrpc.exceptions import JSONRPCDispatchException -from twisted.internet import defer, reactor +from starlette.endpoints import WebSocketEndpoint -from platformio.compat import PY2, dump_json_to_unicode, is_bytes +from platformio.compat import create_task, get_running_loop, is_bytes +from platformio.proc import force_exit -class JSONRPCServerProtocol(WebSocketServerProtocol): - def onOpen(self): - self.factory.connection_nums += 1 - if self.factory.shutdown_timer: - self.factory.shutdown_timer.cancel() - self.factory.shutdown_timer = None +class JSONRPCServerFactoryBase: - def onClose(self, wasClean, code, reason): # pylint: disable=unused-argument - self.factory.connection_nums -= 1 - if self.factory.connection_nums == 0: - self.factory.shutdownByTimeout() - - def onMessage(self, payload, isBinary): # pylint: disable=unused-argument - # click.echo("> %s" % payload) - response = jsonrpc.JSONRPCResponseManager.handle( - payload, self.factory.dispatcher - ).data - # if error - if "result" not in response: - self.sendJSONResponse(response) - return None - - d = defer.maybeDeferred(lambda: response["result"]) - d.addCallback(self._callback, response) - d.addErrback(self._errback, response) - - return None - - def _callback(self, result, response): - response["result"] = result - self.sendJSONResponse(response) - - def _errback(self, failure, response): - if isinstance(failure.value, JSONRPCDispatchException): - e = failure.value - else: - e = JSONRPCDispatchException(code=4999, message=failure.getErrorMessage()) - del response["result"] - response["error"] = e.error._data # pylint: disable=protected-access - self.sendJSONResponse(response) - - def sendJSONResponse(self, response): - # click.echo("< %s" % response) - if "error" in response: - click.secho("Error: %s" % response["error"], fg="red", err=True) - response = dump_json_to_unicode(response) - if not PY2 and not is_bytes(response): - response = response.encode("utf-8") - self.sendMessage(response) - - -class JSONRPCServerFactory(WebSocketServerFactory): - - protocol = JSONRPCServerProtocol connection_nums = 0 - shutdown_timer = 0 + shutdown_timer = None def __init__(self, shutdown_timeout=0): - super(JSONRPCServerFactory, self).__init__() self.shutdown_timeout = shutdown_timeout self.dispatcher = jsonrpc.Dispatcher() - def shutdownByTimeout(self): + def __call__(self, *args, **kwargs): + raise NotImplementedError + + def addHandler(self, handler, namespace): + self.dispatcher.build_method_map(handler, prefix="%s." % namespace) + + def on_client_connect(self): + self.connection_nums += 1 + if self.shutdown_timer: + self.shutdown_timer.cancel() + self.shutdown_timer = None + + def on_client_disconnect(self): + self.connection_nums -= 1 + if self.connection_nums < 1: + self.connection_nums = 0 + + if self.connection_nums == 0: + self.shutdown_by_timeout() + + async def on_shutdown(self): + pass + + def shutdown_by_timeout(self): if self.shutdown_timeout < 1: return def _auto_shutdown_server(): click.echo("Automatically shutdown server on timeout") - reactor.stop() + force_exit() - self.shutdown_timer = reactor.callLater( + self.shutdown_timer = get_running_loop().call_later( self.shutdown_timeout, _auto_shutdown_server ) - def addHandler(self, handler, namespace): - self.dispatcher.build_method_map(handler, prefix="%s." % namespace) + +class WebSocketJSONRPCServerFactory(JSONRPCServerFactoryBase): + def __call__(self, *args, **kwargs): + ws = WebSocketJSONRPCServer(*args, **kwargs) + ws.factory = self + return ws + + +class WebSocketJSONRPCServer(WebSocketEndpoint): + encoding = "text" + factory: WebSocketJSONRPCServerFactory = None + + async def on_connect(self, websocket): + await websocket.accept() + self.factory.on_client_connect() # pylint: disable=no-member + + async def on_receive(self, websocket, data): + create_task(self._handle_rpc(websocket, data)) + + async def on_disconnect(self, websocket, close_code): + self.factory.on_client_disconnect() # pylint: disable=no-member + + async def _handle_rpc(self, websocket, data): + response = jsonrpc.JSONRPCResponseManager.handle( + data, self.factory.dispatcher # pylint: disable=no-member + ) + if response.result and inspect.isawaitable(response.result): + try: + response.result = await response.result + response.data["result"] = response.result + response.error = None + except Exception as exc: # pylint: disable=broad-except + if not isinstance(exc, jsonrpc.exceptions.JSONRPCDispatchException): + exc = jsonrpc.exceptions.JSONRPCDispatchException( + code=4999, message=str(exc) + ) + response.result = None + response.error = exc.error._data # pylint: disable=protected-access + new_data = response.data.copy() + new_data["error"] = response.error + del new_data["result"] + response.data = new_data + + if response.error: + click.secho("Error: %s" % response.error, fg="red", err=True) + if "result" in response.data and is_bytes(response.data["result"]): + response.data["result"] = response.data["result"].decode("utf-8") + + await websocket.send_text(json.dumps(response.data)) diff --git a/platformio/commands/home/run.py b/platformio/commands/home/run.py new file mode 100644 index 0000000000..6e93cc2b46 --- /dev/null +++ b/platformio/commands/home/run.py @@ -0,0 +1,99 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from urllib.parse import urlparse + +import click +import uvicorn +from starlette.applications import Starlette +from starlette.middleware import Middleware +from starlette.responses import PlainTextResponse +from starlette.routing import Mount, Route, WebSocketRoute +from starlette.staticfiles import StaticFiles +from starlette.status import HTTP_403_FORBIDDEN + +from platformio.commands.home.rpc.handlers.account import AccountRPC +from platformio.commands.home.rpc.handlers.app import AppRPC +from platformio.commands.home.rpc.handlers.ide import IDERPC +from platformio.commands.home.rpc.handlers.misc import MiscRPC +from platformio.commands.home.rpc.handlers.os import OSRPC +from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.commands.home.rpc.handlers.project import ProjectRPC +from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory +from platformio.compat import get_running_loop +from platformio.exception import PlatformioException +from platformio.package.manager.core import get_core_package_dir +from platformio.proc import force_exit + + +class ShutdownMiddleware: + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", {}): + await shutdown_server() + await self.app(scope, receive, send) + + +async def shutdown_server(_=None): + get_running_loop().call_later(0.5, force_exit) + return PlainTextResponse("Server has been shutdown!") + + +async def protected_page(_): + return PlainTextResponse( + "Protected PlatformIO Home session", status_code=HTTP_403_FORBIDDEN + ) + + +def run_server(host, port, no_open, shutdown_timeout, home_url): + contrib_dir = get_core_package_dir("contrib-piohome") + if not os.path.isdir(contrib_dir): + raise PlatformioException("Invalid path to PIO Home Contrib") + + ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout) + ws_rpc_factory.addHandler(AccountRPC(), namespace="account") + ws_rpc_factory.addHandler(AppRPC(), namespace="app") + ws_rpc_factory.addHandler(IDERPC(), namespace="ide") + ws_rpc_factory.addHandler(MiscRPC(), namespace="misc") + ws_rpc_factory.addHandler(OSRPC(), namespace="os") + ws_rpc_factory.addHandler(PIOCoreRPC(), namespace="core") + ws_rpc_factory.addHandler(ProjectRPC(), namespace="project") + + path = urlparse(home_url).path + routes = [ + WebSocketRoute(path + "wsrpc", ws_rpc_factory, name="wsrpc"), + Route(path + "__shutdown__", shutdown_server, methods=["POST"]), + Mount(path, StaticFiles(directory=contrib_dir, html=True), name="static"), + ] + if path != "/": + routes.append(Route("/", protected_page)) + + uvicorn.run( + Starlette( + middleware=[Middleware(ShutdownMiddleware)], + routes=routes, + on_startup=[ + lambda: click.echo( + "PIO Home has been started. Press Ctrl+C to shutdown." + ), + lambda: None if no_open else click.launch(home_url), + ], + ), + host=host, + port=port, + log_level="warning", + ) diff --git a/platformio/commands/home/web.py b/platformio/commands/home/web.py deleted file mode 100644 index 32bf069257..0000000000 --- a/platformio/commands/home/web.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.internet import reactor # pylint: disable=import-error -from twisted.web import static # pylint: disable=import-error - - -class WebRoot(static.File): - def render_GET(self, request): - if request.args.get(b"__shutdown__", False): - reactor.stop() - return "Server has been stopped" - - request.setHeader("cache-control", "no-cache, no-store, must-revalidate") - request.setHeader("pragma", "no-cache") - request.setHeader("expires", "0") - return static.File.render_GET(self, request) diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index cb31120566..d0c87d844e 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -13,7 +13,6 @@ # limitations under the License. import json -import os import platform import subprocess import sys @@ -27,11 +26,15 @@ install_completion_code, uninstall_completion_code, ) +from platformio.commands.system.prune import ( + prune_cached_data, + prune_core_packages, + prune_platform_packages, +) from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_cache_dir @click.group("system", short_help="Miscellaneous system commands") @@ -99,22 +102,49 @@ def system_info(json_output): @cli.command("prune", short_help="Remove unused data") @click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation") -def system_prune(force): - click.secho("WARNING! This will remove:", fg="yellow") - click.echo(" - cached API requests") - click.echo(" - cached package downloads") - click.echo(" - temporary data") - if not force: - click.confirm("Do you want to continue?", abort=True) - - reclaimed_total = 0 - cache_dir = get_project_cache_dir() - if os.path.isdir(cache_dir): - reclaimed_total += fs.calculate_folder_size(cache_dir) - fs.rmtree(cache_dir) +@click.option( + "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed" +) +@click.option("--cache", is_flag=True, help="Prune only cached data") +@click.option( + "--core-packages", is_flag=True, help="Prune only unnecessary core packages" +) +@click.option( + "--platform-packages", + is_flag=True, + help="Prune only unnecessary development platform packages", +) +def system_prune(force, dry_run, cache, core_packages, platform_packages): + if dry_run: + click.secho( + "Dry run mode (do not prune, only show data that will be removed)", + fg="yellow", + ) + click.echo() + + reclaimed_cache = 0 + reclaimed_core_packages = 0 + reclaimed_platform_packages = 0 + prune_all = not any([cache, core_packages, platform_packages]) + + if cache or prune_all: + reclaimed_cache = prune_cached_data(force, dry_run) + click.echo() + + if core_packages or prune_all: + reclaimed_core_packages = prune_core_packages(force, dry_run) + click.echo() + + if platform_packages or prune_all: + reclaimed_platform_packages = prune_platform_packages(force, dry_run) + click.echo() click.secho( - "Total reclaimed space: %s" % fs.humanize_file_size(reclaimed_total), fg="green" + "Total reclaimed space: %s" + % fs.humanize_file_size( + reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages + ), + fg="green", ) diff --git a/platformio/commands/system/prune.py b/platformio/commands/system/prune.py new file mode 100644 index 0000000000..e0ef2dd89f --- /dev/null +++ b/platformio/commands/system/prune.py @@ -0,0 +1,98 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from operator import itemgetter + +import click +from tabulate import tabulate + +from platformio import fs +from platformio.package.manager.core import remove_unnecessary_core_packages +from platformio.package.manager.platform import remove_unnecessary_platform_packages +from platformio.project.helpers import get_project_cache_dir + + +def prune_cached_data(force=False, dry_run=False, silent=False): + reclaimed_space = 0 + if not silent: + click.secho("Prune cached data:", bold=True) + click.echo(" - cached API requests") + click.echo(" - cached package downloads") + click.echo(" - temporary data") + cache_dir = get_project_cache_dir() + if os.path.isdir(cache_dir): + reclaimed_space += fs.calculate_folder_size(cache_dir) + if not dry_run: + if not force: + click.confirm("Do you want to continue?", abort=True) + fs.rmtree(cache_dir) + if not silent: + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + return reclaimed_space + + +def prune_core_packages(force=False, dry_run=False, silent=False): + if not silent: + click.secho("Prune unnecessary core packages:", bold=True) + return _prune_packages(force, dry_run, silent, remove_unnecessary_core_packages) + + +def prune_platform_packages(force=False, dry_run=False, silent=False): + if not silent: + click.secho("Prune unnecessary development platform packages:", bold=True) + return _prune_packages(force, dry_run, silent, remove_unnecessary_platform_packages) + + +def _prune_packages(force, dry_run, silent, handler): + if not silent: + click.echo("Calculating...") + items = [ + ( + pkg, + fs.calculate_folder_size(pkg.path), + ) + for pkg in handler(dry_run=True) + ] + items = sorted(items, key=itemgetter(1), reverse=True) + reclaimed_space = sum([item[1] for item in items]) + if items and not silent: + click.echo( + tabulate( + [ + ( + pkg.metadata.spec.humanize(), + str(pkg.metadata.version), + fs.humanize_file_size(size), + ) + for (pkg, size) in items + ], + headers=["Package", "Version", "Size"], + ) + ) + if not dry_run: + if not force: + click.confirm("Do you want to continue?", abort=True) + handler(dry_run=False) + if not silent: + click.secho("Space on disk: %s" % fs.humanize_file_size(reclaimed_space)) + return reclaimed_space + + +def calculate_unnecessary_system_data(): + return ( + prune_cached_data(force=True, dry_run=True, silent=True) + + prune_core_packages(force=True, dry_run=True, silent=True) + + prune_platform_packages(force=True, dry_run=True, silent=True) + ) diff --git a/platformio/compat.py b/platformio/compat.py index 974bdf2f5f..53c1507cfb 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -67,9 +67,9 @@ def ensure_python3(raise_exception=True): return compatible raise UserSideException( "Python 3.6 or later is required for this operation. \n" - "Please install the latest Python 3 and reinstall PlatformIO Core using " - "installation script:\n" - "https://docs.platformio.org/page/core/installation.html" + "Please check a migration guide:\n" + "https://docs.platformio.org/en/latest/core/migration.html" + "#drop-support-for-python-2-and-3-5" ) @@ -78,6 +78,12 @@ def ensure_python3(raise_exception=True): string_types = (str, unicode) + def create_task(coro, name=None): + raise NotImplementedError + + def get_running_loop(): + raise NotImplementedError + def is_bytes(x): return isinstance(x, (buffer, bytearray)) @@ -129,6 +135,12 @@ def load_python_module(name, pathname): import importlib.util from glob import escape as glob_escape + if sys.version_info >= (3, 7): + from asyncio import create_task, get_running_loop + else: + from asyncio import ensure_future as create_task + from asyncio import get_event_loop as get_running_loop + string_types = (str,) def is_bytes(x): diff --git a/platformio/ide/tpls/emacs/.ccls.tpl b/platformio/ide/tpls/emacs/.ccls.tpl index 53c4afebd1..a747bb61b2 100644 --- a/platformio/ide/tpls/emacs/.ccls.tpl +++ b/platformio/ide/tpls/emacs/.ccls.tpl @@ -1,4 +1,4 @@ -{{ cxx_path }} +clang {{"%c"}} {{ !cc_flags }} {{"%cpp"}} {{ !cxx_flags }} diff --git a/platformio/ide/tpls/sublimetext/.ccls.tpl b/platformio/ide/tpls/sublimetext/.ccls.tpl index 53c4afebd1..a747bb61b2 100644 --- a/platformio/ide/tpls/sublimetext/.ccls.tpl +++ b/platformio/ide/tpls/sublimetext/.ccls.tpl @@ -1,4 +1,4 @@ -{{ cxx_path }} +clang {{"%c"}} {{ !cc_flags }} {{"%cpp"}} {{ !cxx_flags }} diff --git a/platformio/ide/tpls/vim/.ccls.tpl b/platformio/ide/tpls/vim/.ccls.tpl index 53c4afebd1..a747bb61b2 100644 --- a/platformio/ide/tpls/vim/.ccls.tpl +++ b/platformio/ide/tpls/vim/.ccls.tpl @@ -1,4 +1,4 @@ -{{ cxx_path }} +clang {{"%c"}} {{ !cc_flags }} {{"%cpp"}} {{ !cxx_flags }} diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 9d2f8245b5..0e101339fb 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -26,6 +26,7 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update +from platformio.commands.system.prune import calculate_unnecessary_system_data from platformio.commands.upgrade import get_latest_version from platformio.compat import ensure_python3 from platformio.package.manager.core import update_core_packages @@ -39,6 +40,8 @@ def on_platformio_start(ctx, force, caller): + ensure_python3(raise_exception=True) + app.set_session_var("command_ctx", ctx) app.set_session_var("force_option", force) set_caller(caller) @@ -46,24 +49,8 @@ def on_platformio_start(ctx, force, caller): if PlatformioCLI.in_silence(): return - after_upgrade(ctx) - if not ensure_python3(raise_exception=False): - click.secho( - """ -Python 2 and Python 3.5 are not compatible with PlatformIO Core 5.0. -Please check the migration guide on how to fix this warning message: -""", - fg="yellow", - ) - click.secho( - "https://docs.platformio.org/en/latest/core/migration.html" - "#drop-support-for-python-2-and-3-5", - fg="blue", - ) - click.echo("") - def on_platformio_end(ctx, result): # pylint: disable=unused-argument if PlatformioCLI.in_silence(): @@ -73,6 +60,7 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_platformio_upgrade() check_internal_updates(ctx, "platforms") check_internal_updates(ctx, "libraries") + check_prune_system() except ( http.HTTPClientError, http.InternetIsOffline, @@ -347,3 +335,31 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches click.echo("*" * terminal_width) click.echo("") + + +def check_prune_system(): + last_check = app.get_state_item("last_check", {}) + interval = 30 * 3600 * 24 # 1 time per month + if (time() - interval) < last_check.get("prune_system", 0): + return + + last_check["prune_system"] = int(time()) + app.set_state_item("last_check", last_check) + threshold_mb = int(app.get_setting("check_prune_system_threshold") or 0) + if threshold_mb <= 0: + return + + unnecessary_mb = calculate_unnecessary_system_data() / 1024 + if unnecessary_mb < threshold_mb: + return + + terminal_width, _ = click.get_terminal_size() + click.echo() + click.echo("*" * terminal_width) + click.secho( + "We found %s of unnecessary PlatformIO system data (temporary files, " + "unnecessary packages, etc.).\nUse `pio system prune --dry-run` to list " + "them or `pio system prune` to save disk space." + % fs.humanize_file_size(unnecessary_mb), + fg="yellow", + ) diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 1487d0bfa4..c81e7186da 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -26,7 +26,10 @@ class PackageManagerUpdateMixin(object): def outdated(self, pkg, spec=None): assert isinstance(pkg, PackageItem) assert not spec or isinstance(spec, PackageSpec) - assert os.path.isdir(pkg.path) and pkg.metadata + assert pkg.metadata + + if not os.path.isdir(pkg.path): + return PackageOutdatedResult(current=pkg.metadata.version) # skip detached package to a specific version detached_conditions = [ diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index a324beb466..24d494b34a 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -27,7 +27,18 @@ from platformio.proc import get_pythonexe_path -def get_core_package_dir(name): +def get_installed_core_packages(): + result = [] + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if pkg: + result.append(pkg) + return result + + +def get_core_package_dir(name, auto_install=True): if name not in __core_packages__: raise exception.PlatformioException("Please upgrade PlatformIO Core") pm = ToolPackageManager() @@ -37,8 +48,10 @@ def get_core_package_dir(name): pkg = pm.get_package(spec) if pkg: return pkg.path + if not auto_install: + return None assert pm.install(spec) - _remove_unnecessary_packages() + remove_unnecessary_core_packages() return pm.get_package(spec).path @@ -52,24 +65,40 @@ def update_core_packages(only_check=False, silent=False): if not silent or pm.outdated(pkg, spec).is_outdated(): pm.update(pkg, spec, only_check=only_check) if not only_check: - _remove_unnecessary_packages() + remove_unnecessary_core_packages() return True -def _remove_unnecessary_packages(): +def remove_unnecessary_core_packages(dry_run=False): + candidates = [] pm = ToolPackageManager() best_pkg_versions = {} + for name, requirements in __core_packages__.items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if not pkg: continue best_pkg_versions[pkg.metadata.name] = pkg.metadata.version + for pkg in pm.get_installed(): - if pkg.metadata.name not in best_pkg_versions: - continue - if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]: - pm.uninstall(pkg) + skip_conds = [ + os.path.isfile(os.path.join(pkg.path, ".piokeep")), + pkg.metadata.spec.owner != "platformio", + pkg.metadata.name not in best_pkg_versions, + pkg.metadata.name in best_pkg_versions + and pkg.metadata.version == best_pkg_versions[pkg.metadata.name], + ] + if not any(skip_conds): + candidates.append(pkg) + + if dry_run: + return candidates + + for pkg in candidates: + pm.uninstall(pkg) + + return candidates def inject_contrib_pysite(verify_openssl=False): @@ -160,7 +189,6 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): pkg.dump_meta() # remove unused files - shutil.rmtree(os.path.join(target_dir, "autobahn", "xbr", "contracts")) for root, dirs, files in os.walk(target_dir): for t in ("_test", "test", "tests"): if t in dirs: @@ -169,19 +197,6 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): if name.endswith((".chm", ".pyc")): os.remove(os.path.join(root, name)) - # apply patches - with open( - os.path.join(target_dir, "autobahn", "twisted", "__init__.py"), "r+" - ) as fp: - contents = fp.read() - contents = contents.replace( - "from autobahn.twisted.wamp import ApplicationSession", - "# from autobahn.twisted.wamp import ApplicationSession", - ) - fp.seek(0) - fp.truncate() - fp.write(contents) - return target_dir @@ -192,8 +207,6 @@ def get_contrib_pysite_deps(): twisted_version = "19.10.0" if PY2 else "20.3.0" result = [ "twisted == %s" % twisted_version, - "autobahn == %s" % ("19.11.2" if PY2 else "20.7.1"), - "json-rpc == 1.13.0", ] # twisted[tls], see setup.py for %twisted_version% @@ -201,14 +214,6 @@ def get_contrib_pysite_deps(): ["pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3"] ) - # zeroconf - if PY2: - result.append( - "https://github.com/ivankravets/python-zeroconf/" "archive/pio-py27.zip" - ) - else: - result.append("zeroconf == 0.26.0") - if "windows" in sys_type: result.append("pypiwin32 == 223") # workaround for twisted wheels diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 9f8bd28ae8..3f77846b7e 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -112,16 +112,11 @@ def install_dependencies(self, pkg, silent=False): ) def _install_dependency(self, dependency, silent=False): - if set(["name", "version"]) <= set(dependency.keys()) and any( - c in dependency["version"] for c in (":", "/", "@") - ): - spec = PackageSpec("%s=%s" % (dependency["name"], dependency["version"])) - else: - spec = PackageSpec( - owner=dependency.get("owner"), - name=dependency.get("name"), - requirements=dependency.get("version"), - ) + spec = PackageSpec( + owner=dependency.get("owner"), + name=dependency.get("name"), + requirements=dependency.get("version"), + ) search_filters = { key: value for key, value in dependency.items() @@ -143,11 +138,12 @@ def uninstall_dependencies(self, pkg, silent=False): if not silent: self.print_message("Removing dependencies...", fg="yellow") for dependency in manifest.get("dependencies"): - pkg = self.get_package( - PackageSpec( - name=dependency.get("name"), requirements=dependency.get("version") - ) + spec = PackageSpec( + owner=dependency.get("owner"), + name=dependency.get("name"), + requirements=dependency.get("version"), ) + pkg = self.get_package(spec) if not pkg: continue self._uninstall(pkg, silent=silent) diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 71e8c5fbfe..efe8a361a5 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os + from platformio import util from platformio.clients.http import HTTPClientError, InternetIsOffline from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.core import get_installed_core_packages from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageType from platformio.platform.exception import IncompatiblePlatform, UnknownBoard @@ -69,7 +72,6 @@ def install( # pylint: disable=arguments-differ, too-many-arguments ) p.install_python_packages() p.on_installed() - self.cleanup_packages(list(p.packages)) return pkg def uninstall(self, spec, silent=False, skip_dependencies=False): @@ -83,7 +85,6 @@ def uninstall(self, spec, silent=False, skip_dependencies=False): if not skip_dependencies: p.uninstall_python_packages() p.on_uninstalled() - self.cleanup_packages(list(p.packages)) return pkg def update( # pylint: disable=arguments-differ, too-many-arguments @@ -118,7 +119,6 @@ def update( # pylint: disable=arguments-differ, too-many-arguments ) p.update_packages(only_check) - self.cleanup_packages(list(p.packages)) if missed_pkgs: p.install_packages( @@ -127,32 +127,6 @@ def update( # pylint: disable=arguments-differ, too-many-arguments return new_pkg or pkg - def cleanup_packages(self, names): - self.memcache_reset() - deppkgs = {} - for platform in PlatformPackageManager().get_installed(): - p = PlatformFactory.new(platform) - for pkg in p.get_installed_packages(): - if pkg.metadata.name not in deppkgs: - deppkgs[pkg.metadata.name] = set() - deppkgs[pkg.metadata.name].add(pkg.metadata.version) - - pm = ToolPackageManager() - for pkg in pm.get_installed(): - if pkg.metadata.name not in names: - continue - if ( - pkg.metadata.name not in deppkgs - or pkg.metadata.version not in deppkgs[pkg.metadata.name] - ): - try: - pm.uninstall(pkg.metadata.spec) - except UnknownPackageError: - pass - - self.memcache_reset() - return True - @util.memoized(expire="5s") def get_installed_boards(self): boards = [] @@ -193,3 +167,37 @@ def board_config(self, id_, platform=None): ): return manifest raise UnknownBoard(id_) + + +# +# Helpers +# + + +def remove_unnecessary_platform_packages(dry_run=False): + candidates = [] + required = set() + core_packages = get_installed_core_packages() + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for pkg in p.get_installed_packages(with_optional=True): + required.add(pkg) + + pm = ToolPackageManager() + for pkg in pm.get_installed(): + skip_conds = [ + pkg.metadata.spec.url, + os.path.isfile(os.path.join(pkg.path, ".piokeep")), + pkg in required, + pkg in core_packages, + ] + if not any(skip_conds): + candidates.append(pkg) + + if dry_run: + return candidates + + for pkg in candidates: + pm.uninstall(pkg) + + return candidates diff --git a/platformio/package/meta.py b/platformio/package/meta.py index edc5d0ffb1..74af191643 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -107,16 +107,21 @@ class PackageSpec(object): # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=redefined-builtin,too-many-arguments self, raw=None, owner=None, id=None, name=None, requirements=None, url=None ): + self._requirements = None self.owner = owner self.id = id self.name = name - self._requirements = None self.url = url self.raw = raw if requirements: - self.requirements = requirements + try: + self.requirements = requirements + except ValueError as exc: + if not self.name or self.url or self.raw: + raise exc + self.raw = "%s=%s" % (self.name, requirements) self._name_is_custom = False - self._parse(raw) + self._parse(self.raw) def __eq__(self, other): return all( @@ -405,7 +410,12 @@ def __repr__(self): ) def __eq__(self, other): - return all([self.path == other.path, self.metadata == other.metadata]) + if not self.path or not other.path: + return self.path == other.path + return os.path.realpath(self.path) == os.path.realpath(other.path) + + def __hash__(self): + return hash(os.path.realpath(self.path)) def exists(self): return os.path.isdir(self.path) diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index ac495b4827..786f1efc4e 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -17,18 +17,17 @@ class PlatformPackagesMixin(object): - def get_package_spec(self, name): - version = self.packages[name].get("version", "") - if any(c in version for c in (":", "/", "@")): - return PackageSpec("%s=%s" % (name, version)) + def get_package_spec(self, name, version=None): return PackageSpec( - owner=self.packages[name].get("owner"), name=name, requirements=version + owner=self.packages[name].get("owner"), + name=name, + requirements=version or self.packages[name].get("version"), ) - def get_package(self, name): + def get_package(self, name, spec=None): if not name: return None - return self.pm.get_package(self.get_package_spec(name)) + return self.pm.get_package(spec or self.get_package_spec(name)) def get_package_dir(self, name): pkg = self.get_package(name) @@ -38,12 +37,18 @@ def get_package_version(self, name): pkg = self.get_package(name) return str(pkg.metadata.version) if pkg else None - def get_installed_packages(self): + def get_installed_packages(self, with_optional=False): result = [] - for name in self.packages: - pkg = self.get_package(name) - if pkg: - result.append(pkg) + for name, options in self.packages.items(): + versions = [options.get("version")] + if with_optional: + versions.extend(options.get("optionalVersions", [])) + for version in versions: + if not version: + continue + pkg = self.get_package(name, self.get_package_spec(name, version)) + if pkg: + result.append(pkg) return result def dump_used_packages(self): diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index 0f2bd15f34..1aff6709ba 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -37,6 +37,8 @@ def load_module(name, path): @classmethod def new(cls, pkg_or_spec): + # pylint: disable=import-outside-toplevel + platform_dir = None platform_name = None if isinstance(pkg_or_spec, PackageItem): @@ -45,9 +47,7 @@ def new(cls, pkg_or_spec): elif os.path.isdir(pkg_or_spec): platform_dir = pkg_or_spec else: - from platformio.package.manager.platform import ( # pylint: disable=import-outside-toplevel - PlatformPackageManager, - ) + from platformio.package.manager.platform import PlatformPackageManager pkg = PlatformPackageManager().get_package(pkg_or_spec) if not pkg: diff --git a/platformio/proc.py b/platformio/proc.py index 8db7153e53..24640c3818 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -24,6 +24,7 @@ WINDOWS, get_filesystem_encoding, get_locale_encoding, + get_running_loop, string_types, ) @@ -31,7 +32,10 @@ class AsyncPipeBase(object): def __init__(self): self._fd_read, self._fd_write = os.pipe() - self._pipe_reader = os.fdopen(self._fd_read) + if PY2: + self._pipe_reader = os.fdopen(self._fd_read) + else: + self._pipe_reader = os.fdopen(self._fd_read, errors="backslashreplace") self._buffer = "" self._thread = Thread(target=self.run) self._thread.start() @@ -67,10 +71,10 @@ def do_reading(self): line = "" print_immediately = False - for byte in iter(lambda: self._pipe_reader.read(1), ""): - self._buffer += byte + for char in iter(lambda: self._pipe_reader.read(1), ""): + self._buffer += char - if line and byte.strip() and line[-3:] == (byte * 3): + if line and char.strip() and line[-3:] == (char * 3): print_immediately = True if print_immediately: @@ -78,12 +82,12 @@ def do_reading(self): if line: self.data_callback(line) line = "" - self.data_callback(byte) - if byte == "\n": + self.data_callback(char) + if char == "\n": print_immediately = False else: - line += byte - if byte != "\n": + line += char + if char != "\n": continue self.line_callback(line) line = "" @@ -214,3 +218,12 @@ def append_env_path(name, value): return cur_value os.environ[name] = os.pathsep.join([cur_value, value]) return os.environ[name] + + +def force_exit(code=0): + try: + get_running_loop().stop() + except: # pylint: disable=bare-except + pass + finally: + sys.exit(code) diff --git a/platformio/project/options.py b/platformio/project/options.py index b5eaf337d2..213a2104ef 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -681,6 +681,11 @@ def ConfigEnvOption(*args, **kwargs): "network address)" ), ), + ConfigEnvOption( + group="debug", + name="debug_speed", + description="A debug adapter speed (JTAG speed)", + ), ConfigEnvOption( group="debug", name="debug_svd_path", diff --git a/platformio/util.py b/platformio/util.py index e777d10a02..6b1af88614 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -25,7 +25,7 @@ import click -from platformio import __version__, exception, proc +from platformio import __version__, compat, exception, proc from platformio.compat import PY2, WINDOWS from platformio.fs import cd, load_json # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import @@ -162,14 +162,10 @@ def get_logical_devices(): def get_mdns_services(): - # pylint: disable=import-outside-toplevel - try: - import zeroconf - except ImportError: - from platformio.package.manager.core import inject_contrib_pysite + compat.ensure_python3() - inject_contrib_pysite() - import zeroconf # pylint: disable=import-outside-toplevel + # pylint: disable=import-outside-toplevel + import zeroconf class mDNSListener(object): def __init__(self): @@ -178,15 +174,20 @@ def __init__(self): self._found_services = [] def __enter__(self): - zeroconf.ServiceBrowser(self._zc, "_services._dns-sd._udp.local.", self) + zeroconf.ServiceBrowser( + self._zc, + [ + "_http._tcp.local.", + "_hap._tcp.local.", + "_services._dns-sd._udp.local.", + ], + self, + ) return self def __exit__(self, etype, value, traceback): self._zc.close() - def remove_service(self, zc, type_, name): - pass - def add_service(self, zc, type_, name): try: assert zeroconf.service_type_name(name) @@ -201,6 +202,12 @@ def add_service(self, zc, type_, name): if s: self._found_services.append(s) + def remove_service(self, zc, type_, name): + pass + + def update_service(self, zc, type_, name): + pass + def get_services(self): return self._found_services @@ -225,12 +232,7 @@ def get_services(self): { "type": service.type, "name": service.name, - "ip": ".".join( - [ - str(c if isinstance(c, int) else ord(c)) - for c in service.address - ] - ), + "ip": ", ".join(service.parsed_addresses()), "port": service.port, "properties": properties, } diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index b04b946f4e..86edd5b054 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -167,3 +167,6 @@ ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666", ENV{ID_MM_DEVICE # CMSIS-DAP compatible adapters ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + +# Atmel AVR Dragon +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2107", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" diff --git a/setup.py b/setup.py index 6472eadbe2..ad251a89bb 100644 --- a/setup.py +++ b/setup.py @@ -26,18 +26,28 @@ from platformio.compat import PY2, WINDOWS -install_requires = [ - "bottle<0.13", +minimal_requirements = [ + "bottle==0.12.*", "click>=5,<8%s" % (",!=7.1,!=7.1.1" if WINDOWS else ""), "colorama", - "pyserial>=3,<4,!=3.3", - "requests>=2.4.0,<3", - "semantic_version>=2.8.1,<3", - "tabulate>=0.8.3,<1", - "pyelftools>=0.25,<1", - "marshmallow%s" % (">=2,<3" if PY2 else ">=2"), + "marshmallow%s" % (">=2,<3" if PY2 else ">=2,<4"), + "pyelftools>=0.27,<1", + "pyserial==3.*", + "requests==2.*", + "semantic_version==2.8.*", + "tabulate==0.8.*", ] +if not PY2: + minimal_requirements.append("zeroconf==0.28.*") + +home_requirements = [ + "aiofiles==0.6.*", + "json-rpc==1.13.*", + "starlette==0.14.*", + "uvicorn==0.13.*", + "wsproto==1.0.*", +] setup( name=__title__, @@ -48,10 +58,7 @@ author_email=__email__, url=__url__, license=__license__, - python_requires=", ".join( - [">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"] - ), - install_requires=install_requires, + install_requires=minimal_requirements + ([] if PY2 else home_requirements), packages=find_packages(exclude=["tests.*", "tests"]) + ["scripts"], package_data={ "platformio": [ @@ -77,7 +84,6 @@ "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python", - "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Topic :: Software Development", "Topic :: Software Development :: Build Tools", diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index b8c8e65a0e..c631a613c9 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -154,7 +154,7 @@ def test_check_includes_passed(clirunner, check_dir): inc_count = l.count("-I") # at least 1 include path for default mode - assert inc_count > 1 + assert inc_count > 0 def test_check_silent_mode(clirunner, validate_cliresult, check_dir): diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index 32b3d56a90..1cda6409c0 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -169,6 +169,15 @@ def test_spec_vcs_urls(): url="git+git@github.com:platformio/platformio-core.git", requirements="^1.2.3,!=5", ) + assert PackageSpec( + owner="platformio", + name="external-repo", + requirements="https://github.com/platformio/platformio-core", + ) == PackageSpec( + owner="platformio", + name="external-repo", + url="git+https://github.com/platformio/platformio-core", + ) def test_spec_as_dict(): diff --git a/tox.ini b/tox.ini index bb41a67b04..8a1f0b04db 100644 --- a/tox.ini +++ b/tox.ini @@ -13,14 +13,14 @@ # limitations under the License. [tox] -envlist = py27,py37,py38,py39 +envlist = py36,py37,py38,py39 [testenv] passenv = * usedevelop = True deps = - py36,py37,py38,py39: black - isort<5 + black + isort pylint pytest pytest-xdist