NEW: Better package splitting for multi-output recipes with negative glob in outputs/files

conda_build/build.py

@@ -1692,7 +1692,14 @@ def post_process_files(m: MetaData, initial_prefix_files):
     return new_files
 
 
-def bundle_conda(output, metadata: MetaData, env, stats, **kw):
+def bundle_conda(
+    output,
+    metadata: MetaData,
+    env,
+    stats,
+    new_prefix_files: set[str] = set(),
+    **kw,
+):
     log = utils.get_logger(__name__)
     log.info("Packaging %s", metadata.dist())
     get_all_replacements(metadata.config)
@@ -1802,10 +1809,26 @@ def bundle_conda(output, metadata: MetaData, env, stats, **kw):
     if files:
         # Files is specified by the output
         # we exclude the list of files that we want to keep, so post-process picks them up as "new"
-        keep_files = {
-            os.path.normpath(pth)
-            for pth in utils.expand_globs(files, metadata.config.host_prefix)
-        }
+        if isinstance(files, dict):
+            # When file matching with include/exclude lists, only
+            # new_prefix_files are considered. Files in the PREFIX from other
+            # recipes (dependencies) are ignored
+            include = files.get("include") or []
+            exclude = files.get("exclude") or []
+            exclude_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(exclude, metadata.config.host_prefix)
+            }
+            keep_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(include, metadata.config.host_prefix)
+            }
+            keep_files = new_prefix_files.intersection(keep_files) - exclude_files
+        else:
+            keep_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(files, metadata.config.host_prefix)
+            }
         pfx_files = set(utils.prefix_files(metadata.config.host_prefix))
         initial_files = {
             item
@@ -1980,7 +2003,13 @@ def bundle_conda(output, metadata: MetaData, env, stats, **kw):
     return final_outputs
 
 
-def bundle_wheel(output, metadata: MetaData, env, stats):
+def bundle_wheel(
+    output,
+    metadata: MetaData,
+    env,
+    stats,
+    new_prefix_files: set[str] = set(),
+):
     ext = ".bat" if utils.on_win else ".sh"
     with TemporaryDirectory() as tmpdir, utils.tmp_chdir(metadata.config.work_dir):
         dest_file = os.path.join(metadata.config.work_dir, "wheel_output" + ext)
@@ -2706,7 +2735,11 @@ def build(
                     # This is wrong, files has not been expanded at this time and could contain
                     # wildcards.  Also well, I just do not understand this, because when this
                     # does contain wildcards, the files in to_remove will slip back in.
-                    if "files" in output_d:
+                    if (
+                        "files" in output_d
+                        and output_d["files"] is not None
+                        and not isinstance(output_d["files"], dict)
+                    ):
                         output_d["files"] = set(output_d["files"]) - to_remove
 
                     # copies the backed-up new prefix files into the newly created host env
@@ -2722,7 +2755,9 @@ def build(
                     with utils.path_prepended(m.config.build_prefix):
                         env = environ.get_dict(m=m)
                     pkg_type = "conda" if not hasattr(m, "type") else m.type
-                    newly_built_packages = bundlers[pkg_type](output_d, m, env, stats)
+                    newly_built_packages = bundlers[pkg_type](
+                        output_d, m, env, stats, new_prefix_files
+                    )
                     # warn about overlapping files.
                     if "checksums" in output_d:
                         for file, csum in output_d["checksums"].items():

docs/source/resources/define-metadata.rst

@@ -1313,7 +1313,10 @@ build prefix. Explicit file lists support glob expressions.
 Directory names are also supported, and they recursively include
 contents.
 
-.. code-block:: none
+.. warning::
+   When defining `outputs/files` as a list, any file in the prefix (including those installed by host dependencies) matching one of the glob expressions is included in the output.
+
+.. code-block:: yaml
 
    outputs:
      - name: subpackage-name
@@ -1323,6 +1326,29 @@ contents.
          - *.some-extension
          - somefolder/*.some-extension
 
+Greater control over file matching may be
+achieved by defining ``files`` as a dictionary separating files to
+``include`` from those to ``exclude``.
+When using include/exclude, only files installed by
+the current recipe are considered. i.e. files in the prefix installed
+by host dependencies are excluded. include/exclude may not be used
+simultaneously with glob expressions listed directly in ``outputs/files``.
+Files matching both include and exclude expressions will be excluded.
+
+.. code-block:: yaml
+
+   outputs:
+     - name: subpackage-name
+       files:
+         include:
+           - a-file
+           - a-folder
+           - *.some-extension
+           - somefolder/*.some-extension
+         exclude:
+           - *.exclude-extension
+           - a-folder/**/*.some-extension
+
 Scripts that create or move files into the build prefix can be
 any kind of script. Known script types need only specify the
 script name. Currently the list of recognized extensions is
@@ -1372,10 +1398,9 @@ A subpackage does not automatically inherit any dependencies from its top-level
 recipe, so any build or run requirements needed by the subpackage must be
 explicitly specified.
 
-.. code-block:: none
+.. code-block:: yaml
 
    outputs:
-
      - name: subpackage-name
        requirements:
          build:

news/5216-files-exclude

@@ -0,0 +1,19 @@
+### Enhancements
+
+* Add new include/exclude sections for glob expressions in multi-output `outputs/files`. (#4196 via #5216)
+
+### Bug fixes
+
+* <news item>
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>

tests/test-recipes/split-packages/copying_files/bld.bat

@@ -2,4 +2,14 @@ echo "weee" > %PREFIX%\subpackage_file1
 mkdir %PREFIX%\somedir
 echo "weee" > %PREFIX%\somedir\subpackage_file1
 echo "weee" > %PREFIX%\subpackage_file1.ext
-echo "weee" > %PREFIX%\subpackage_file2.ext
+echo "weee" > %PREFIX%\subpackage_file2.ext
+echo "weee" > %PREFIX%\subpackage_file3.ext
+
+echo "weee" > %PREFIX%\subpackage_include_exclude1
+mkdir %PREFIX%\anotherdir
+echo "weee" > %PREFIX%\anotherdir\subpackage_include_exclude1
+echo "weee" > %PREFIX%\subpackage_include_exclude1.wav
+echo "weee" > %PREFIX%\subpackage_include_exclude2.wav
+echo "weee" > %PREFIX%\subpackage_include_exclude3.wav
+mkdir %PREFIX%\Library\bin
+echo "weee" > %PREFIX%\Library\bin\dav1d.fake

tests/test-recipes/split-packages/copying_files/build.sh

@@ -6,3 +6,18 @@ echo "weee" > $PREFIX/somedir/subpackage_file1
 # test glob patterns
 echo "weee" > $PREFIX/subpackage_file1.ext
 echo "weee" > $PREFIX/subpackage_file2.ext
+echo "weee" > $PREFIX/subpackage_file3.ext
+
+# The files used to test the two subpackages must be disjoint because they are
+# coinstalled
+# test copying filename
+echo "weee" > $PREFIX/subpackage_include_exclude1
+# test copying by folder name
+mkdir $PREFIX/anotherdir
+echo "weee" > $PREFIX/anotherdir/subpackage_include_exclude1
+# test glob patterns
+echo "weee" > $PREFIX/subpackage_include_exclude1.wav
+echo "weee" > $PREFIX/subpackage_include_exclude2.wav
+echo "weee" > $PREFIX/subpackage_include_exclude3.wav
+mkdir $PREFIX/lib
+echo "weee" > $PREFIX/lib/libdav1d.fake

tests/test-recipes/split-packages/copying_files/meta.yaml

@@ -4,14 +4,44 @@ package:
 
 requirements:
   run:
-    - my_script_subpackage
+    - my_script_subpackage_files
+    - my_script_subpackage_include_exclude
 
 outputs:
-  - name: my_script_subpackage
+  - name: my_script_subpackage_files
+    build:
+      ignore_run_exports_from:
+        - libpng
+    requirements:
+      host:
+        - libpng=1.6.39
     files:
       - subpackage_file1
       - somedir
       - "*.ext"
+      # Libs should match because they are in the prefix
+      - "lib/libpng*"       # [unix]
+      - "Library/bin/libpng*"  # [win]
+    test:
+      script: subpackage_test.py
+      script_interpreter: python
+  - name: my_script_subpackage_include_exclude
+    build:
+      ignore_run_exports_from:
+        - dav1d
+    requirements:
+      host:
+        - dav1d=1.2.1
+    files:
+      include:
+        - subpackage_include_exclude1
+        - anotherdir
+        - "*.wav"
+        # Libs should not match because they come from a different package
+        - "lib/libdav1d*"       # [unix]
+        - "Library/bin/dav1d*"  # [win]
+      exclude:
+        - "*3.wav"
     test:
       script: subpackage_test.py
       script_interpreter: python

tests/test-recipes/split-packages/copying_files/subpackage_test.py

@@ -1,33 +1,76 @@
 import os
+import sys
 
-print(os.getenv('PREFIX'))
-filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1')
-assert os.path.isfile(filename)
+if os.getenv("PKG_NAME") == "my_script_subpackage_files":
+    file_basename = "subpackage_file"
+    dirname = "somedir"
+    extension = "ext"
+
+    if "darwin" in sys.platform:
+        external_host_file = "lib/libpng16.dylib"
+    elif "win32" in sys.platform:
+        external_host_file = "Library/bin/libpng16.dll"
+    else:
+        external_host_file = "lib/libpng16.so"
+
+    filename = os.path.join(os.environ["PREFIX"], f"{file_basename}3.{extension}")
+    print(filename)
+    assert os.path.isfile(filename), filename + " is missing"
+    print("glob files OK")
+
+    filename = os.path.join(os.environ["PREFIX"], external_host_file)
+    print(filename)
+    assert os.path.isfile(filename), filename + " is missing"
+    print("glob files prefix OK")
+
+if os.getenv("PKG_NAME") == "my_script_subpackage_include_exclude":
+    file_basename = "subpackage_include_exclude"
+    dirname = "anotherdir"
+    extension = "wav"
+
+    if "darwin" in sys.platform:
+        external_host_file = "lib/libdav1d.6.dylib"
+    elif "win32" in sys.platform:
+        external_host_file = "Library/bin/dav1d.dll"
+    else:
+        external_host_file = "lib/libdav1d.so.6"
+
+    filename = os.path.join(os.environ["PREFIX"], f"{file_basename}3.{extension}")
+    assert not os.path.isfile(filename), filename + " is missing"
+    print("glob exclude OK")
+
+    filename = os.path.join(os.environ["PREFIX"], external_host_file)
+    assert not os.path.isfile(filename), filename + " is missing"
+    print("glob exclude prefix OK")
+
+print(os.getenv("PREFIX"))
+filename = os.path.join(os.environ["PREFIX"], f"{file_basename}1")
+assert os.path.isfile(filename), filename + " is missing"
 contents = open(filename).read().rstrip()
-if hasattr(contents, 'decode'):
+if hasattr(contents, "decode"):
     contents = contents.decode()
-assert "weee" in contents, 'incorrect file contents: %s' % contents
+assert "weee" in contents, "incorrect file contents: %s" % contents
 print("plain file OK")
 
-filename = os.path.join(os.environ['PREFIX'], 'somedir', 'subpackage_file1')
+filename = os.path.join(os.environ["PREFIX"], dirname, f"{file_basename}1")
 assert os.path.isfile(filename), filename + " is missing"
 contents = open(filename).read().rstrip()
-if hasattr(contents, 'decode'):
+if hasattr(contents, "decode"):
     contents = contents.decode()
-assert "weee" in contents, 'incorrect file contents: %s' % contents
+assert "weee" in contents, "incorrect file contents: %s" % contents
 print("subfolder file OK")
 
-filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1.ext')
-assert os.path.isfile(filename)
+filename = os.path.join(os.environ["PREFIX"], f"{file_basename}1.{extension}")
+assert os.path.isfile(filename), filename + " is missing"
 contents = open(filename).read().rstrip()
-if hasattr(contents, 'decode'):
+if hasattr(contents, "decode"):
     contents = contents.decode()
-assert "weee" in contents, 'incorrect file contents: %s' % contents
+assert "weee" in contents, "incorrect file contents: %s" % contents
 
-filename = os.path.join(os.environ['PREFIX'], 'subpackage_file2.ext')
-assert os.path.isfile(filename)
+filename = os.path.join(os.environ["PREFIX"], f"{file_basename}2.{extension}")
+assert os.path.isfile(filename), filename + " is missing"
 contents = open(filename).read().rstrip()
-if hasattr(contents, 'decode'):
+if hasattr(contents, "decode"):
     contents = contents.decode()
-assert "weee" in contents, 'incorrect file contents: %s' % contents
+assert "weee" in contents, "incorrect file contents: %s" % contents
 print("glob OK")

tests/test_subpackages.py

@@ -18,6 +18,7 @@
 
 
 @pytest.mark.slow
+@pytest.mark.serial
 @pytest.mark.parametrize(
     "recipe",
     [