Add a --parallel flag to the install command

poetry/console/commands/install.py

@@ -22,6 +22,9 @@ class InstallCommand(EnvCommand):
         option(
             "remove-untracked", None, "Removes packages not present in the lock file.",
         ),
+        option(
+            "parallel", None, "Installs packages in parallel when it is safe to do so.",
+        ),
         option(
             "extras",
             "E",
@@ -61,6 +64,7 @@ def handle(self):
         installer.dev_mode(not self.option("no-dev"))
         installer.dry_run(self.option("dry-run"))
         installer.remove_untracked(self.option("remove-untracked"))
+        installer.parallel(self.option("parallel"))
         installer.verbose(self.option("verbose"))
 
         return_code = installer.run()

poetry/installation/executor.py

@@ -0,0 +1,127 @@
+from concurrent.futures import ThreadPoolExecutor
+from threading import Lock
+
+from poetry.puzzle.operations import Install
+from poetry.puzzle.operations import Uninstall
+from poetry.puzzle.operations import Update
+from poetry.puzzle.operations.operation import Operation
+
+
+class Executor:
+    def __init__(self, installer, parallel, io, execute_operations, dry_run, verbose):
+        self._installer = installer
+        self._parallel = parallel
+        self._io = io
+        self._execute_operations = execute_operations
+        self._dry_run = dry_run
+        self._verbose = verbose
+
+        self._write_lock = Lock()
+
+    def execute(self, ops):
+        if self._parallel:
+            executor = ThreadPoolExecutor()
+        else:
+            executor = ThreadPoolExecutor(max_workers=1)
+        current_depth = None
+        current_tasks = []
+        for op in ops:
+            if current_depth not in (None, op.depth):
+                for future in current_tasks:
+                    future.result()
+            current_depth = op.depth
+            current_tasks.append(executor.submit(self._execute_single, op))
+        for future in current_tasks:
+            future.result()
+
+    def _execute_single(self, operation):  # type: (Operation) -> None
+        """
+        Execute a given operation.
+        """
+        method = operation.job_type
+
+        getattr(self, "_execute_{}".format(method))(operation)
+
+    def _execute_update(self, operation):  # type: (Update) -> None
+        source = operation.initial_package
+        target = operation.target_package
+
+        if operation.skipped:
+            if self._verbose and (self._execute_operations or self._dry_run):
+                self._write_line(
+                    "  - Skipping <c1>{}</c1> (<c2>{}</c2>) {}".format(
+                        target.pretty_name,
+                        target.full_pretty_version,
+                        operation.skip_reason,
+                    )
+                )
+
+            return
+
+        if self._execute_operations or self._dry_run:
+            self._write_line(
+                "  - Updating <c1>{}</c1> (<c2>{}</c2> -> <c2>{}</c2>)".format(
+                    target.pretty_name,
+                    source.full_pretty_version,
+                    target.full_pretty_version,
+                )
+            )
+
+        if not self._execute_operations:
+            return
+
+        self._installer.update(source, target)
+
+    def _execute_install(self, operation):  # type: (Install) -> None
+        if operation.skipped:
+            if self._verbose and (self._execute_operations or self.is_dry_run()):
+                self._write_line(
+                    "  - Skipping <c1>{}</c1> (<c2>{}</c2>) {}".format(
+                        operation.package.pretty_name,
+                        operation.package.full_pretty_version,
+                        operation.skip_reason,
+                    )
+                )
+
+            return
+
+        if self._execute_operations or self.is_dry_run():
+            self._write_line(
+                "  - Installing <c1>{}</c1> (<c2>{}</c2>)".format(
+                    operation.package.pretty_name, operation.package.full_pretty_version
+                )
+            )
+
+        if not self._execute_operations:
+            return
+
+        self._installer.install(operation.package)
+
+    def _execute_uninstall(self, operation):  # type: (Uninstall) -> None
+        if operation.skipped:
+            if self._verbose and (self._execute_operations or self._dry_run):
+                self._write_line(
+                    "  - Not removing <c1>{}</c1> (<c2>{}</c2>) {}".format(
+                        operation.package.pretty_name,
+                        operation.package.full_pretty_version,
+                        operation.skip_reason,
+                    )
+                )
+
+            return
+
+        if self._execute_operations or self._dry_run:
+            self._write_line(
+                "  - Removing <c1>{}</c1> (<c2>{}</c2>)".format(
+                    operation.package.pretty_name, operation.package.full_pretty_version
+                )
+            )
+
+        if not self._execute_operations:
+            return
+
+        self._installer.remove(operation.package)
+
+    def _write_line(self, *args, **kwargs):
+        with self._write_lock:
+            self._io.write_line(*args, **kwargs)

poetry/installation/installer.py

@@ -18,6 +18,7 @@
 from poetry.utils.helpers import canonicalize_name
 
 from .base_installer import BaseInstaller
+from .executor import Executor
 from .pip_installer import PipInstaller
 
 
@@ -39,6 +40,7 @@ def __init__(
 
         self._dry_run = False
         self._remove_untracked = False
+        self._parallel = False
         self._update = False
         self._verbose = False
         self._write_lock = True
@@ -88,9 +90,17 @@ def remove_untracked(self, remove_untracked=True):  # type: (bool) -> Installer
 
         return self
 
+    def parallel(self, parallel=True):  # type: (bool) -> Installer
+        self._parallel = parallel
+
+        return self
+
     def is_remove_untracked(self):  # type: () -> bool
         return self._remove_untracked
 
+    def is_parallel(self):  # type: () -> bool
+        return self._parallel
+
     def verbose(self, verbose=True):  # type: (bool) -> Installer
         self._verbose = verbose
 
@@ -300,8 +310,15 @@ def _do_install(self, local_repo):
             )
 
         self._io.write_line("")
-        for op in ops:
-            self._execute(op)
+        executor = Executor(
+            installer=self._installer,
+            parallel=self._parallel,
+            io=self._io,
+            execute_operations=self._execute_operations,
+            dry_run=self.is_dry_run(),
+            verbose=self.is_verbose(),
+        )
+        executor.execute(ops)
 
     def _write_lock_file(self, repo):  # type: (Repository) -> None
         if self._update and self._write_lock:
@@ -311,94 +328,6 @@ def _write_lock_file(self, repo):  # type: (Repository) -> None
                 self._io.write_line("")
                 self._io.write_line("<info>Writing lock file</>")
 
-    def _execute(self, operation):  # type: (Operation) -> None
-        """
-        Execute a given operation.
-        """
-        method = operation.job_type
-
-        getattr(self, "_execute_{}".format(method))(operation)
-
-    def _execute_install(self, operation):  # type: (Install) -> None
-        if operation.skipped:
-            if self.is_verbose() and (self._execute_operations or self.is_dry_run()):
-                self._io.write_line(
-                    "  - Skipping <c1>{}</c1> (<c2>{}</c2>) {}".format(
-                        operation.package.pretty_name,
-                        operation.package.full_pretty_version,
-                        operation.skip_reason,
-                    )
-                )
-
-            return
-
-        if self._execute_operations or self.is_dry_run():
-            self._io.write_line(
-                "  - Installing <c1>{}</c1> (<c2>{}</c2>)".format(
-                    operation.package.pretty_name, operation.package.full_pretty_version
-                )
-            )
-
-        if not self._execute_operations:
-            return
-
-        self._installer.install(operation.package)
-
-    def _execute_update(self, operation):  # type: (Update) -> None
-        source = operation.initial_package
-        target = operation.target_package
-
-        if operation.skipped:
-            if self.is_verbose() and (self._execute_operations or self.is_dry_run()):
-                self._io.write_line(
-                    "  - Skipping <c1>{}</c1> (<c2>{}</c2>) {}".format(
-                        target.pretty_name,
-                        target.full_pretty_version,
-                        operation.skip_reason,
-                    )
-                )
-
-            return
-
-        if self._execute_operations or self.is_dry_run():
-            self._io.write_line(
-                "  - Updating <c1>{}</c1> (<c2>{}</c2> -> <c2>{}</c2>)".format(
-                    target.pretty_name,
-                    source.full_pretty_version,
-                    target.full_pretty_version,
-                )
-            )
-
-        if not self._execute_operations:
-            return
-
-        self._installer.update(source, target)
-
-    def _execute_uninstall(self, operation):  # type: (Uninstall) -> None
-        if operation.skipped:
-            if self.is_verbose() and (self._execute_operations or self.is_dry_run()):
-                self._io.write_line(
-                    "  - Not removing <c1>{}</c1> (<c2>{}</c2>) {}".format(
-                        operation.package.pretty_name,
-                        operation.package.full_pretty_version,
-                        operation.skip_reason,
-                    )
-                )
-
-            return
-
-        if self._execute_operations or self.is_dry_run():
-            self._io.write_line(
-                "  - Removing <c1>{}</c1> (<c2>{}</c2>)".format(
-                    operation.package.pretty_name, operation.package.full_pretty_version
-                )
-            )
-
-        if not self._execute_operations:
-            return
-
-        self._installer.remove(operation.package)
-
     def _populate_local_repo(self, local_repo, ops):
         for op in ops:
             if isinstance(op, Uninstall):

poetry/puzzle/operations/operation.py

@@ -9,6 +9,7 @@ def __init__(self, reason=None):  # type: (Union[str, None]) -> None
 
         self._skipped = False
         self._skip_reason = None
+        self._depth = -1
 
     @property
     def job_type(self):  # type: () -> str
@@ -30,6 +31,10 @@ def skip_reason(self):  # type: () -> Union[str, None]
     def package(self):
         raise NotImplementedError()
 
+    @property
+    def depth(self):
+        return self._depth
+
     def format_version(self, package):  # type: (...) -> str
         return package.full_pretty_version
 
@@ -44,3 +49,6 @@ def unskip(self):  # type: () -> Operation
         self._skip_reason = None
 
         return self
+
+    def set_depth(self, depth):
+        self._depth = depth

poetry/puzzle/solver.py

@@ -158,13 +158,18 @@ def solve(self, use_latest=None):  # type: (...) -> List[Operation]
                 if installed.name not in locked_names:
                     operations.append(Uninstall(installed))
 
+        for op in operations:
+            # Packages to be uninstalled have no depth so we default to 0
+            # since it actually doesn't matter since removals are always on top.
+            op.set_depth(
+                depths[packages.index(op.package)] if op.job_type != "uninstall" else 0
+            )
+
         return sorted(
             operations,
             key=lambda o: (
                 o.job_type == "uninstall",
-                # Packages to be uninstalled have no depth so we default to 0
-                # since it actually doesn't matter since removals are always on top.
-                -depths[packages.index(o.package)] if o.job_type != "uninstall" else 0,
+                -o.depth,
                 o.package.name,
                 o.package.version,
             ),