Added import-check command, tests, and doc.

closes #7549

CHANGES/7549.feature

@@ -0,0 +1 @@
+Added /importers/core/pulp/import-check/ to validate import-parameters.

docs/workflows/import-export.rst

@@ -364,3 +364,30 @@ The command to create an import will return a task that can be used to monitor t
 also see a history of past imports::
 
     http :/pulp/api/v3/importers/core/pulp/f8acba87-0250-4640-b56b-c92597d344b7/imports/
+
+Pre-validating import parameters
+--------------------------------
+
+There are a number of things that can keep an import from being successful, ranging from a specified
+export-file not being available to bad JSON specified for ``repo_mapping``. You can pre-validate your
+proposed import using the ``import-check`` command::
+
+    http POST :/pulp/api/v3/importers/core/pulp/import-check/ \
+      path=/tmp/export-file-path toc=/tmp/export-toc-path repo_mapping:="{\"source\": \"dest\"}"
+
+``import-check`` will validate that:
+
+    * paths are in ``ALLOWED_IMPORT_PATHS``
+    * containing directory exists
+    * containing directory is readable
+    * path/toc file(s) exist and are readable
+    * for TOC, containing directory is writeable
+    * repo_mapping is valid JSON
+
+``import-check`` is a low-overhead synchronous call. It does not attempt to do validations that
+require database access or long-running tasks such as verifying checksums. All parameters are optional. 
+
+.. note::
+
+    For ``path`` and ``toc``, if the ALLOWED_IMPORT_PATHS check fails, no further information will be given.
+

pulpcore/app/serializers/__init__.py

@@ -41,16 +41,19 @@
     SingleArtifactContentSerializer,
 )
 from .exporter import (  # noqa
-    ExportSerializer,
     ExporterSerializer,
+    ExportSerializer,
     FilesystemExporterSerializer,
     PublicationExportSerializer,
     PulpExporterSerializer,
     PulpExportSerializer,
 )
 from .importer import (  # noqa
-    ImportSerializer,
+    EvaluationSerializer,
     ImporterSerializer,
+    ImportSerializer,
+    PulpImportCheckResponseSerializer,
+    PulpImportCheckSerializer,
     PulpImporterSerializer,
     PulpImportSerializer,
 )
@@ -83,5 +86,4 @@
     UploadSerializer,
     UploadDetailSerializer,
 )
-
 from .user import GroupSerializer, UserSerializer  # noqa

pulpcore/app/serializers/importer.py

@@ -10,6 +10,7 @@
     ImportIdentityField,
     ModelSerializer,
     RelatedField,
+    ValidateFieldsMixin,
 )
 
 
@@ -131,21 +132,16 @@ def validate_toc(self, value):
         """
         Check validity of provided 'toc' parameter.
 
-        'toc' must:
-          * be within ALLOWED_IMPORT_PATHS.
-          * be valid JSON
-          * point to chunked-export-files that exist 'next to' the 'toc' file
+        'toc' must be within ALLOWED_IMPORT_PATHS.
 
-        NOTE: this method does NOT validate checksums of the chunked-export-files. That
+        NOTE: this method does NOT validate existence/sanity of export-files. That
         happens asynchronously, due to time/responsiveness constraints.
 
         Args:
             value (str): The user-provided toc-file-path to be validated.
 
         Raises:
-            ValidationError: When toc is not in the ALLOWED_IMPORT_PATHS setting,
-            toc is not a valid JSON table-of-contents file, or when toc points to
-            chunked-export-files that can't be found in the same directory as the toc-file.
+            ValidationError: When toc is not in the ALLOWED_IMPORT_PATHS setting
 
         Returns:
             The validated value.
@@ -170,3 +166,76 @@ class Meta:
             "path",
             "toc",
         )
+
+
+class EvaluationSerializer(serializers.Serializer):
+    """
+    Results from evaluating a proposed parameter to a PulpImport call.
+    """
+
+    context = serializers.CharField(
+        help_text=_("Parameter value being evaluated."),
+    )
+    is_valid = serializers.BooleanField(
+        help_text=_("True if evaluation passed, false otherwise."),
+    )
+    messages = serializers.ListField(
+        child=serializers.CharField(),
+        help_text=_("Messages describing results of all evaluations done. May be an empty list."),
+    )
+
+
+class PulpImportCheckResponseSerializer(serializers.Serializer):
+    """
+    Return the response to a PulpImport import-check call.
+    """
+
+    toc = EvaluationSerializer(
+        help_text=_("Evaluation of proposed 'toc' file for PulpImport"),
+        required=False,
+    )
+    path = EvaluationSerializer(
+        help_text=_("Evaluation of proposed 'path' file for PulpImport"),
+        required=False,
+    )
+    repo_mapping = EvaluationSerializer(
+        help_text=_("Evaluation of proposed 'repo_mapping' file for PulpImport"),
+        required=False,
+    )
+
+
+class PulpImportCheckSerializer(ValidateFieldsMixin, serializers.Serializer):
+    """
+    Check validity of provided import-options.
+
+    Provides the ability to check that an import is 'sane' without having to actually
+    create an importer.
+    """
+
+    path = serializers.CharField(
+        help_text=_("Path to export-tar-gz that will be imported."), required=False
+    )
+    toc = serializers.CharField(
+        help_text=_(
+            "Path to a table-of-contents file describing chunks to be validated, "
+            "reassembled, and imported."
+        ),
+        required=False,
+    )
+    repo_mapping = serializers.CharField(
+        help_text=_(
+            "Mapping of repo names in an export file to the repo names in Pulp. "
+            "For example, if the export has a repo named 'foo' and the repo to "
+            "import content into was 'bar', the mapping would be \"{'foo': 'bar'}\"."
+        ),
+        required=False,
+    )
+
+    def validate(self, data):
+        data = super().validate(data)
+        if "path" not in data and "toc" not in data and "repo_mapping" not in data:
+            raise serializers.ValidationError(
+                _("One of 'path', 'toc', or 'repo_mapping' must be specified.")
+            )
+        else:
+            return data

pulpcore/app/urls.py

@@ -15,7 +15,7 @@
 from rest_framework_nested import routers
 
 from pulpcore.app.apps import pulp_plugin_configs
-from pulpcore.app.views import OrphansView, RepairView, StatusView
+from pulpcore.app.views import OrphansView, PulpImporterImportCheckView, RepairView, StatusView
 from pulpcore.constants import API_ROOT
 from pulpcore.openapi import PulpSchemaGenerator
 
@@ -123,6 +123,10 @@ def __repr__(self):
     url(r"^{api_root}repair/".format(api_root=API_ROOT), RepairView.as_view()),
     url(r"^{api_root}status/".format(api_root=API_ROOT), StatusView.as_view()),
     url(r"^{api_root}orphans/".format(api_root=API_ROOT), OrphansView.as_view()),
+    url(
+        r"^{api_root}importers/core/pulp/import-check/".format(api_root=API_ROOT),
+        PulpImporterImportCheckView.as_view(),
+    ),
     url(r"^auth/", include("rest_framework.urls")),
     path(settings.ADMIN_SITE_URL, admin.site.urls),
 ]

pulpcore/app/views/__init__.py

@@ -1,3 +1,4 @@
 from .orphans import OrphansView  # noqa
 from .status import StatusView  # noqa
 from .repair import RepairView  # noqa
+from .importer import PulpImporterImportCheckView  # noqa

pulpcore/app/views/importer.py

@@ -0,0 +1,124 @@
+from gettext import gettext as _
+import json
+import os
+from drf_spectacular.utils import extend_schema
+from rest_framework.views import APIView
+from rest_framework.response import Response
+
+from pulpcore.app import settings
+from pulpcore.app.serializers import PulpImportCheckResponseSerializer, PulpImportCheckSerializer
+
+
+def _check_allowed_import_path(a_path):
+    user_provided_realpath = os.path.realpath(a_path)
+    for allowed_path in settings.ALLOWED_IMPORT_PATHS:
+        if user_provided_realpath.startswith(allowed_path):
+            return True, None
+    return False, _(
+        "{} is not an allowed import path".format(os.path.dirname(os.path.realpath(a_path)))
+    )
+
+
+def _validate_file(in_param, data):
+    """
+    Returns a (is-valid, msgs[]) tuple describing all problems found with data[in_param]
+
+    We check for a number of things, attempting to return all the errors we can find. We don't want
+    to give out information for files in arbitrary locations on the filesystem; if the check
+    for ALLOWED_IMPORT_PATHS fails, we report that and ignore any other problems.
+
+    If the directory containing the base-file doesn't exist, or isn't readable, or the specified
+    file doesn't exist, report and return.
+
+    Error-messages for all other checks are additive.
+    """
+    # check allowed, leave if failed
+    file = data[in_param]
+    real_file = os.path.realpath(file)
+    rc, msg = _check_allowed_import_path(real_file)
+    if not rc:
+        return rc, [msg]
+
+    # check directory-sanity, leave if failed
+    owning_dir = os.path.dirname(real_file)
+    if not os.path.exists(owning_dir):
+        return False, [_("directory {} does not exist").format(owning_dir)]
+    if not os.access(owning_dir, os.R_OK):
+        return False, [_("directory {} does not allow read-access").format(owning_dir)]
+
+    # check file-exists, leave if failed
+    if not os.path.exists(real_file):
+        return False, [_("file {} does not exist").format(real_file)]
+
+    # check file-sanity
+    msgs = []
+    isfile = os.path.isfile(real_file)
+    readable = os.access(real_file, os.R_OK)
+
+    rc = isfile and readable
+    if not isfile:
+        msgs.append(_("{} is not a file".format(real_file)))
+    if not readable:
+        msgs.append(_("{} exists but cannot be read".format(real_file)))
+
+    # extra check for toc-dir-write
+    if in_param == "toc":
+        if not os.access(owning_dir, os.W_OK):
+            rc = False
+            msgs.append(_("directory {} must allow pulp write-access".format(owning_dir)))
+
+    return rc, msgs
+
+
+class PulpImporterImportCheckView(APIView):
+    """
+    Returns validity of proposed parameters for a PulpImport call.
+    """
+
+    @extend_schema(
+        summary="Validate the parameters to be used for a PulpImport call",
+        operation_id="pulp_import_check_post",
+        request=PulpImportCheckSerializer,
+        responses={200: PulpImportCheckResponseSerializer},
+    )
+    def post(self, request, format=None):
+        """
+        Evaluates validity of proposed PulpImport parameters 'toc', 'path', and 'repo_mapping'.
+
+        * Checks that toc, path are in ALLOWED_IMPORT_PATHS
+        * if ALLOWED:
+          * Checks that toc, path exist and are readable
+          * If toc specified, checks that containing dir is writeable
+        * Checks that repo_mapping is valid JSON
+        """
+        serializer = PulpImportCheckSerializer(data=request.data)
+        if serializer.is_valid():
+            data = {}
+            if "toc" in serializer.data:
+                data["toc"] = {}
+                data["toc"]["context"] = serializer.data["toc"]
+                data["toc"]["is_valid"], data["toc"]["messages"] = _validate_file(
+                    "toc", serializer.data
+                )
+
+            if "path" in serializer.data:
+                data["path"] = {}
+                data["path"]["context"] = serializer.data["path"]
+                data["path"]["is_valid"], data["path"]["messages"] = _validate_file(
+                    "path", serializer.data
+                )
+
+            if "repo_mapping" in serializer.data:
+                data["repo_mapping"] = {}
+                data["repo_mapping"]["context"] = serializer.data["repo_mapping"]
+                try:
+                    json.loads(serializer.data["repo_mapping"])
+                    data["repo_mapping"]["is_valid"] = True
+                    data["repo_mapping"]["messages"] = []
+                except json.JSONDecodeError:
+                    data["repo_mapping"]["is_valid"] = False
+                    data["repo_mapping"]["messages"] = [_("invalid JSON")]
+
+            crs = PulpImportCheckResponseSerializer(data, context={"request": request})
+            return Response(crs.data)
+        return Response(serializer.errors, status=400)