diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index e906464..34e889f 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -19,12 +19,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install mamba and dependencies
-        uses: mamba-org/provision-with-micromamba@main
+      - uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: ci_env.yml
-          environment-name: ci-env
-          micromamba-version: '1.4.1'
+          cache-environment: true
+
 
       - name: Lint check
         shell: bash -l {0}
diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml
index 1485e4d..b8b9771 100644
--- a/.github/workflows/workflow.yaml
+++ b/.github/workflows/workflow.yaml
@@ -28,12 +28,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install mamba and dependencies
-        uses: mamba-org/provision-with-micromamba@main
+      - uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: ci_env.yml
-          environment-name: ci-env
-          micromamba-version: '1.4.1'
+          cache-environment: true
 
       - name: Install empack
         shell: bash -l {0}
diff --git a/config/empack_config.yaml b/config/empack_config.yaml
index 41a8b2d..f42bfe9 100644
--- a/config/empack_config.yaml
+++ b/config/empack_config.yaml
@@ -1,183 +1,39 @@
 packages:
   setuptools:
-    include_patterns:
-      - pattern: '**/pkg_resources/*.so'
-      - pattern: '**/pkg_resources/**/*.so'
-      - pattern: '**/pkg_resources/*.py'
-      - pattern: '**/pkg_resources/**/*.py'
-      - pattern: '**/*.dist-info/METADATA'
-  bokeh:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '**/bokeh/**/*.html'
-      - pattern: '**/bokeh/**/*.js'
-      - pattern: '**/bokeh/*.json'
-      - pattern: '**/*.dist-info/METADATA'
     exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  plotly:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '**/plotly/package_data/**'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  folium:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: '**/folium/templates/**'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  branca:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: '**/branca/templates/**'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  python-dateutil:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: '*dateutil-zoneinfo.tar.gz'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  # For matplotlib <3.5.2 which didn't have matplotlib-base published
-  matplotlib:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: '**/matplotlib/mpl-data/**'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  matplotlib-base:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: '**/matplotlib/mpl-data/**'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  scikit-learn:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: '**/sklearn/datasets/**'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  scikit-image:
-    include_patterns:
-      - pattern: '*.txt'
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: '**/skimage/data/**'
-      - pattern: '**/skimage/io/_plugins/*.ini'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-  xarray:
-    include_patterns:
-      - pattern: '**/*.py'
-      - pattern: '**/static/css/*.css'
-      - pattern: '**/static/html/*.html'
-      - pattern: '**/*.dist-info/METADATA'
+      - pattern: '**/_distutils_hack/**'
+      - pattern: '**/distutils-precedence.pth'
+      - pattern: '**/setuptools/**'
+      - pattern: '**/pkg_resources/_vendor/**'
+  zlib:
     exclude_patterns:
-      - pattern: '**/tests/**/*.py'
+      - pattern: '**/so'
   widgetsnbextension:
     exclude_patterns:
       - pattern: '**'
-  python:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: 'share/zoneinfo/**'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-      - pattern: '**/tests/*.py'
-      - pattern: '**/tests/*.so'
-  python-tzdata:
-    include_patterns:
-      - pattern: '*.so'
-      - pattern: '*.py'
-      - pattern: '*.json'
-      - pattern: 'share/zoneinfo/**'
-      - pattern: '**/tzdata/zoneinfo/**'
-      - pattern: '**/*.dist-info/METADATA'
-    exclude_patterns:
-      - pattern: '**/tests/**/*.py'
-      - pattern: '**/tests/**/*.so'
-      - pattern: '**/tests/*.py'
-      - pattern: '**/tests/*.so'
-      - pattern: '**/tzdata/zoneinfo/**/*.pyc'
-  zlib:
-    exclude_patterns:
-      - pattern: '**/so'
-  itables:
-    include_patterns:
-      - pattern: '*.py'
-      - pattern: 'html/**'
-      - pattern: 'external/**'
-      - pattern: '**/*.dist-info/METADATA'
-  urllib3:
-    include_patterns:
-      - pattern: '*.py'
-      - pattern: '**/contrib/emscripten/emscripten_fetch_worker.js'
-  certifi:
-    include_patterns:
-      - pattern: '*.py'
-      - pattern: '*.pem'
-  pyvis:
-    include_patterns:
-      - pattern: '*.py'
-      - pattern: '*.html'
-  pint:
-     include_patterns:
-       - pattern: '*.so'
-       - pattern: '*.py'
-       - pattern: '*.txt'
-     exclude_patterns:
-       - pattern: '**/tests/**/*.py'
-       - pattern: '**/tests/**/*.so'
 default:
-  include_patterns:
-    - pattern: '*.so'
-    - pattern: '*.py'
-    - pattern: '*.json'
-    - pattern: 'share/zoneinfo/**'
-    - pattern: '**/*.dist-info/METADATA'
   exclude_patterns:
-    - pattern: '**/tests/**/*.py'
-    - pattern: '**/tests/**/*.so'
-    - pattern: '**/tests/*.py'
-    - pattern: '**/tests/*.so'
-    - pattern: 'share/jupyter/nbextensions/**/*'
-    - pattern: 'share/jupyter/labextensions/**/*'
-    - pattern: 'etc/jupyter/nbconfig/**/*'
+    - pattern: 'include/**'
+    - pattern: '**/include/**'
+    - pattern: 'bin/**'
+    - pattern: '**/bin/**'
+    - pattern: 'tests/**'
+    - pattern: '**/tests/**'
+    - pattern: '**/*.ini'
+    - pattern: '**/*.exe'
+    - pattern: '**/*.a'
+    - pattern: '**/*.c'
+    - pattern: '**/*.pxd'
+    - pattern: '**/*.pyi'
+    - pattern: '**/*.pyx'
+    - pattern: '**/*.pyc'
+    - pattern: '**/*.typed'
+    - pattern: '**/*.egg-info'
+    - pattern: '**/site-packages/pip/_vendor/**'
+    - pattern: '**/idlelib/**'
+    - pattern: '**/ensurepip/**'
     - pattern: 'Scripts/*'
+    # Excluding jupyter front-end and server extensions
+    - pattern: 'share/jupyter/nbextensions/**'
+    - pattern: 'share/jupyter/labextensions/**'
+    - pattern: 'etc/jupyter/nbconfig/**'
diff --git a/empack/file_patterns.py b/empack/file_patterns.py
index c27cec5..f4c8db6 100644
--- a/empack/file_patterns.py
+++ b/empack/file_patterns.py
@@ -21,7 +21,7 @@ def match(self, path):
 
 
 class FileFilter:
-    def __init__(self, include_patterns=None, exclude_patterns=None):
+    def __init__(self, exclude_patterns=None):
         def patter_from_dict(**d):
             if "pattern" in d:
                 return UnixPattern(**d)
@@ -30,33 +30,31 @@ def patter_from_dict(**d):
             else:
                 raise ValueError("pattern or regex must be provided")
 
-        if include_patterns is None:
-            include_patterns = []
         if exclude_patterns is None:
             exclude_patterns = []
-        self.include_patterns = [patter_from_dict(**p) for p in include_patterns]
         self.exclude_patterns = [patter_from_dict(**p) for p in exclude_patterns]
 
     def match(self, path):
-        include = False
-        for ip in self.include_patterns:
-            if ip.match(path):
-                include = True
-        if include:
-            for ep in self.exclude_patterns:
-                if ep.match(path):
-                    return False
-        return include
+        return all(not ep.match(path) for ep in self.exclude_patterns)
 
 
 class PkgFileFilter:
-    def __init__(self, packages, default=None):
+    def __init__(self, packages=None, default=None):
         self.packages = {}
+
+        if packages is None:
+            packages = {}
+
+        default_exclude_patterns = []
+        if default is not None and "exclude_patterns" in default:
+            default_exclude_patterns = default["exclude_patterns"]
+
         for k, v in packages.items():
             if isinstance(v, dict):
-                self.packages[k] = FileFilter(**v)
-            elif isinstance(v, list):
-                self.packages[k] = [FileFilter(**x) for x in v]
+                exclude_patterns = default_exclude_patterns
+                if "exclude_patterns" in v:
+                    exclude_patterns = exclude_patterns + v["exclude_patterns"]
+                self.packages[k] = FileFilter(exclude_patterns=exclude_patterns)
             else:
                 err = f"invalid value for package {k}: {v}"
                 raise ValueError(err)
diff --git a/empack/filter_env.py b/empack/filter_env.py
index 9465857..5e31e5e 100644
--- a/empack/filter_env.py
+++ b/empack/filter_env.py
@@ -92,14 +92,18 @@ def filter_pkg(env_prefix, pkg_meta, target_dir, matchers):
         for _i, matcher in enumerate(matchers):
             include = matcher.match(path=file)
             if include:
-                included.append(file)
                 path = env_path / file
                 if path.is_symlink() and not path.exists():
                     continue
 
                 dest_fpath = os.path.join(target_dir, file)
                 os.makedirs(os.path.dirname(dest_fpath), exist_ok=True)
-                shutil.copy(os.path.join(env_prefix, file), dest_fpath)
+                try:
+                    shutil.copy(os.path.join(env_prefix, file), dest_fpath)
+                    included.append(file)
+                except FileNotFoundError:
+                    # This may happen when following a symlink on a filtered out file
+                    pass
                 break
     path = write_minimal_conda_meta(pkg_meta=pkg_meta, env_prefix=target_dir)
     included.append(path.relative_to(target_dir))
diff --git a/tests/empack_test_config.yaml b/tests/empack_test_config.yaml
index 4ee31fe..7ba46e0 100644
--- a/tests/empack_test_config.yaml
+++ b/tests/empack_test_config.yaml
@@ -1,27 +1,5 @@
-packages:
-  python-dateutil:
-    include_patterns:
-      - regex: '^(?!.*\/tests\/).*(.*.\.py$)|(.*.\.so$)|(.*dateutil-zoneinfo\.tar\.gz$)'
-  matplotlib:
-    include_patterns:
-      - regex: '^(?!.*\/tests\/).*(.*.\.py$)|(.*.\.so$)'
-      - pattern: "*matplotlibrc"
-
-  scikit-learn:
-    include_patterns:
-      - regex: '^(?!.*\/tests\/).*(.*.\.py$)|(.*.\.so$)'
-      - pattern: "**/sklearn/datasets/**"
-
-  scikit-image:
-    include_patterns:
-      - regex: '^(?!.*\/tests\/).*(.*.\.py$)|(.*.\.so$)'
-      - pattern: "**/skimage/data/**"
-
-
 default:
-  include_patterns:
-    - pattern: '*.so'
-    - pattern: '*.py'
   exclude_patterns:
+    - pattern: '**/*.pyc'
     - pattern: '**/tests/**/*.py'
     - pattern: '**/tests/**/*.so'
diff --git a/tests/empack_test_extra_config.yaml b/tests/empack_test_extra_config.yaml
index bf24dc2..dc3e129 100644
--- a/tests/empack_test_extra_config.yaml
+++ b/tests/empack_test_extra_config.yaml
@@ -1,13 +1,11 @@
 packages:
   scikit-image:
     exclude_patterns:
-      - pattern: '**'
+      - pattern: '**/bar.py'
+      - pattern: '**/bar.so'
 
-
-  scipy:
-    - include_patterns:
-      - pattern: '**/core'
-    - include_patterns:
-      - pattern: '**/sparse'
-    - include_patterns:
-      - pattern: '**'
+default:
+  exclude_patterns:
+    - pattern: '**/*.pyc'
+    - pattern: '**/tests/**/*.py'
+    - pattern: '**/tests/**/*.so'
diff --git a/tests/test_filter.py b/tests/test_filter.py
index 382d15d..7c85a9c 100644
--- a/tests/test_filter.py
+++ b/tests/test_filter.py
@@ -13,11 +13,6 @@ def test_unix_pattern():
 
 def test_file_filter():
     fp = FileFilter(
-        include_patterns=[
-            dict(pattern="*.py"),
-            dict(pattern="*.so"),
-            dict(pattern="*matplotlibrc"),
-        ],
         exclude_patterns=[dict(pattern="**/tests/*")],
     )
     assert fp.match(
@@ -33,23 +28,22 @@ def test_file_filter():
 
 
 def test_empty_file_filter():
-    fp = FileFilter(include_patterns=[], exclude_patterns=[])
-    assert not fp.match("/home/fu/bar.py")
-    assert not fp.match("/home/fu/bar.so")
-    assert not fp.match("/home/tests/fu/bar.py")
-    assert not fp.match("/home/tests/fu/bar.so")
-    assert not fp.match("/hometests/fu/bar.py")
-    assert not fp.match("/hometests/fu/bar.so")
+    fp = FileFilter(exclude_patterns=[])
+    assert fp.match("/home/fu/bar.py")
+    assert fp.match("/home/fu/bar.so")
+    assert fp.match("/home/tests/fu/bar.py")
+    assert fp.match("/home/tests/fu/bar.so")
+    assert fp.match("/hometests/fu/bar.py")
+    assert fp.match("/hometests/fu/bar.so")
 
 
 def test_dataset_filter():
     fp = FileFilter(
-        include_patterns=[dict(pattern="**/sklearn/datasets/**")],
         exclude_patterns=[],
     )
     assert fp.match("/home/fu/sklearn/datasets/some/folder.txt")
     assert fp.match("/home/fu/sklearn/datasets/some/folder.py")
-    assert not fp.match("/home/fu/sxklearn/datasets/some/folder.txt")
+    assert fp.match("/home/fu/sxklearn/datasets/some/folder.txt")
 
 
 def test_from_yaml():
diff --git a/tests/test_integration.py b/tests/test_integration.py
index 9b40ba6..d274249 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -7,7 +7,7 @@
 
 from .conftest import CHANNELS
 
-PYJS_VERSION = "1.0.0"
+PYJS_VERSION = "2.1.0"
 PYJS_SPEC = f"pyjs>={PYJS_VERSION}"