use build_log_writer

conda-store-server/conda_store_server/build.py

@@ -156,17 +156,36 @@ def build_environment(conda_command, environment_filename, conda_prefix):
             "-f",
             str(environment_filename),
         ],
-        stderr=subprocess.STDOUT,
+        stdout=build_log_writer,
+        stderr=build_log_writer,
         encoding="utf-8",
     )
 
 
-def build_lock_environment(lock_filename: pathlib.Path, conda_prefix: pathlib.Path):
-    return subprocess.check_output(
+def build_lock_environment(
+    lock_filename: pathlib.Path, conda_prefix: pathlib.Path, build_log_writer=None
+):
+    build_log_writer = build_log_writer or sys.stdout
+    info = subprocess.run(
+        ["conda", "info"],
+        stderr=build_log_writer,
+        stdout=build_log_writer,
+    )
+    info.check_returncode()
+    build_log_writer.write(f"{'='*100}\n{'='*100}".encode("utf-8"))
+    config = subprocess.run(
+        ["conda", "config", "--show"],
+        stderr=build_log_writer,
+        stdout=build_log_writer,
+    )
+    config.check_returncode()
+    build_log_writer.write(f"{'='*100}\n{'='*100}".encode("utf-8"))
+    build_process = subprocess.Popen(
         ["conda-lock", "install", "--prefix", str(conda_prefix), str(lock_filename)],
-        stderr=subprocess.STDOUT,
-        encoding="utf-8",
+        stderr=build_log_writer,
+        stdout=build_log_writer,
     )
+    return build_process
 
 
 def fetch_and_extract_packages(conda_store, conda_lock_filename: pathlib.Path):
@@ -265,6 +284,8 @@ def build_conda_environment(conda_store, build):
             with utils.timer(conda_store.log, f"building {conda_prefix}"):
                 tmp_environment_filename = os.path.join(tmpdir, "environment.yaml")
                 tmp_lock_filename = os.path.join(tmpdir, "conda-lock.yml")
+                build_log_writer = pathlib.Path(tmpdir, "build.log").open("wb")
+                build_log_reader = pathlib.Path(tmpdir, "build.log").open("rb")
 
                 with open(tmp_environment_filename, "w") as f:
                     yaml.dump(build.specification.spec, f)
@@ -277,10 +298,19 @@ def build_conda_environment(conda_store, build):
 
                 fetch_and_extract_packages(conda_store, pathlib.Path(tmp_lock_filename))
 
-                output = build_lock_environment(
+                build_process = build_lock_environment(
                     pathlib.Path(tmp_lock_filename),
                     conda_prefix,
+                    build_log_writer,
                 )
+                # A blocking wait could be replaced with asynchronously reading from build_log_reader
+                # See the 3rd part of this answer (https://stackoverflow.com/a/18422264)
+                build_process.wait()
+                output = build_log_reader.read()
+                if build_process.returncode:
+                    raise subprocess.CalledProcessError(
+                        build_process.returncode, build_process.args, output
+                    )
 
                 if build.specification.spec.get("variables") is not None:
                     set_conda_environment_variables(
@@ -296,9 +326,9 @@ def build_conda_environment(conda_store, build):
             uid = conda_store.default_uid
             gid = conda_store.default_gid
 
-            if permissions is not None and oct(stat.S_IMODE(stat_info.st_mode))[-3:] != str(
-                permissions
-            ):
+            if permissions is not None and oct(stat.S_IMODE(stat_info.st_mode))[
+                -3:
+            ] != str(permissions):
                 conda_store.log.info(
                     f"modifying permissions of {conda_prefix} to permissions={permissions}"
                 )
@@ -308,7 +338,10 @@ def build_conda_environment(conda_store, build):
             if (
                 uid is not None
                 and gid is not None
-                and (str(uid) != str(stat_info.st_uid) or str(gid) != str(stat_info.st_gid))
+                and (
+                    str(uid) != str(stat_info.st_uid)
+                    or str(gid) != str(stat_info.st_gid)
+                )
             ):
                 conda_store.log.info(
                     f"modifying permissions of {conda_prefix} to uid={uid} and gid={gid}"
@@ -319,7 +352,7 @@ def build_conda_environment(conda_store, build):
             packages = conda.conda_prefix_packages(conda_prefix)
             build.size = utils.disk_usage(conda_prefix)
 
-            set_build_completed(conda_store, build, output.encode("utf-8"), packages)
+            set_build_completed(conda_store, build, output, packages)
         except subprocess.CalledProcessError as e:
             conda_store.log.exception(e)
             set_build_failed(conda_store, build, e.output.encode("utf-8"))