diff --git a/pandas/tests/io/pytables/test_file_handling.py b/pandas/tests/io/pytables/test_file_handling.py
index 1a126ad75c01c..623d6e664090f 100644
--- a/pandas/tests/io/pytables/test_file_handling.py
+++ b/pandas/tests/io/pytables/test_file_handling.py
@@ -230,11 +230,11 @@ def test_complibs_default_settings_override(tmp_path, setup_path):
 @pytest.mark.parametrize("lvl", range(10))
 @pytest.mark.parametrize("lib", tables.filters.all_complibs)
 @pytest.mark.filterwarnings("ignore:object name is not a valid")
-@pytest.mark.xfail(
+@pytest.mark.skipif(
     not PY311 and is_ci_environment() and is_platform_linux(),
-    reason="producing invalid start bytes",
-    raises=UnicodeDecodeError,
-    strict=False,
+    reason="Segfaulting in a CI environment"
+    # with xfail, would sometimes raise UnicodeDecodeError
+    # invalid state byte
 )
 def test_complibs(tmp_path, lvl, lib):
     # GH14478