feat: csv loader updates for multi course runs of External LOBs

course_discovery/apps/course_metadata/data_loaders/constants.py

@@ -12,6 +12,7 @@ class CSVIngestionErrors:
     MISSING_COURSE_RUN_TYPE = 'MISSING_COURSE_RUN_TYPE'
     MISSING_REQUIRED_DATA = 'MISSING_REQUIRED_DATA'
     COURSE_CREATE_ERROR = 'COURSE_CREATE_ERROR'
+    COURSE_RUN_CREATE_ERROR = 'COURSE_RUN_CREATE_ERROR'
     IMAGE_DOWNLOAD_FAILURE = 'IMAGE_DOWNLOAD_FAILURE'
     LOGO_IMAGE_DOWNLOAD_FAILURE = 'LOGO_IMAGE_DOWNLOAD_FAILURE'
     COURSE_UPDATE_ERROR = 'COURSE_UPDATE_ERROR'
@@ -37,6 +38,10 @@ class CSVIngestionErrorMessages:
     COURSE_CREATE_ERROR = '[COURSE_CREATE_ERROR] Unable to create course {course_title} in the system. The ingestion ' \
                           'failed with the exception: {exception_message}'
 
+    COURSE_RUN_CREATE_ERROR = '[COURSE_RUN_CREATE_ERROR] Unable to create course run of the course {course_title} ' \
+                              'with variant_id {variant_id} in the system. The ingestion failed with the exception: ' \
+                              '{exception_message}'
+
     COURSE_UPDATE_ERROR = '[COURSE_UPDATE_ERROR] Unable to update course {course_title} in the system. The update ' \
                           'failed with the exception: {exception_message}'
 

course_discovery/apps/course_metadata/data_loaders/csv_loader.py

@@ -168,15 +168,22 @@ def ingest(self):  # pylint: disable=too-many-statements
             course_key = self.get_course_key(org_key, row['number'])
             course = Course.objects.filter_drafts(key=course_key, partner=self.partner).first()
             is_course_created = False
+            is_course_run_created = False
 
             if course:
-                course_run = CourseRun.objects.filter_drafts(course=course).first()
-                logger.info("Course {} is located in the database.".format(course_key))  # lint-amnesty, pylint: disable=logging-format-interpolation
+                try:
+                    logger.info("Course {} is located in the database.".format(course_key))  # lint-amnesty, pylint: disable=logging-format-interpolation
+                    course_run, is_course_run_created = self._get_or_create_course_run(
+                        row, course, course_type, course_run_type.uuid
+                    )
+                except Exception as exc:
+                    logger.exception(exc)
+                    continue
             else:
                 logger.info("Course key {} could not be found in database, creating the course.".format(course_key))  # lint-amnesty, pylint: disable=logging-format-interpolation
                 try:
                     _ = self._create_course(row, course_type, course_run_type.uuid)
-                except Exception as exc:  # pylint: disable=broad-except
+                except Exception as exc:
                     exception_message = exc
                     if hasattr(exc, 'response'):
                         exception_message = exc.response.content.decode('utf-8')
@@ -191,6 +198,7 @@ def ingest(self):  # pylint: disable=too-many-statements
                 course = Course.everything.get(key=course_key, partner=self.partner)
                 course_run = CourseRun.everything.filter(course=course).first()
                 is_course_created = True
+                is_course_run_created = True
 
             is_downloaded = download_and_save_course_image(
                 course,
@@ -204,7 +212,7 @@ def ingest(self):  # pylint: disable=too-many-statements
             if not is_course_created:
                 self.add_product_source(course)
 
-            is_draft = self.get_draft_flag(course_run)
+            is_draft = self.get_draft_flag(course=course)
             logger.info(f"Draft flag is set to {is_draft} for the course {course_title}")
 
             try:
@@ -260,7 +268,7 @@ def ingest(self):  # pylint: disable=too-many-statements
             logger.info("Course and course run updated successfully for course key {}".format(course_key))  # lint-amnesty, pylint: disable=logging-format-interpolation
             self.course_uuids[str(course.uuid)] = course_title
             self._register_successful_ingestion(
-                str(course.uuid), is_course_created, course.active_url_slug,
+                str(course.uuid), is_course_created, is_course_run_created, course.active_url_slug,
                 row.get('external_course_marketing_type', None))
 
         self._archive_stale_products(course_external_identifiers)
@@ -269,6 +277,60 @@ def ingest(self):  # pylint: disable=too-many-statements
         self._render_error_logs()
         self._render_course_uuids()
 
+    def _get_or_create_course_run(self, data, course, course_type, course_run_type_uuid):
+        """
+        Helper method to get or create a course run for external LOB courses.
+
+        This method will first try to find a course run with the given variant_id and if it does not find one,
+        it will try to find a course run with the given start and end date. If it does not find a course run with
+        either of these, it will create a new course run.
+
+        Args:
+            data(dict): Dictionary containing the course run data
+            course(Course): Course object
+            course_type(CourseType): CourseType object
+            course_run_type_uuid(uuid): UUID of the course run type
+
+        Returns:
+            course_run(CourseRun): CourseRun object
+            is_course_run_created(bool): Boolean indicating if a new course run was created
+        """
+        is_course_run_created = False
+
+        course_runs = CourseRun.objects.filter_drafts(course=course)
+        variant_id = data.get('variant_id', '')
+        start_datetime = self.get_formatted_datetime_string(f"{data['start_date']} {data['start_time']}")
+        end_datetime = self.get_formatted_datetime_string(f'{data["end_date"]} {data["end_time"]}')
+
+        filtered_course_runs = course_runs.filter(
+            Q(variant_id=variant_id) | (Q(start=start_datetime) & Q(end=end_datetime))
+        ).order_by('created')
+        course_run = filtered_course_runs.last()
+
+        if not course_run:
+            logger.info(
+                f'Course Run with variant_id {variant_id} could not be found.'
+                f'Creating new course run for course {course.key} with variant_id {variant_id}'
+            )
+            try:
+                last_run = course_runs.last()
+                _ = self._create_course_run(data, course, course_type, course_run_type_uuid, last_run.key)
+                is_course_run_created = True
+            except Exception as exc:  # pylint: disable=broad-except
+                exception_message = exc
+                if hasattr(exc, 'response'):
+                    exception_message = exc.response.content.decode('utf-8')
+                error_message = CSVIngestionErrorMessages.COURSE_RUN_CREATE_ERROR.format(
+                    course_title=course.title,
+                    variant_id=variant_id,
+                    exception_message=exception_message
+                )
+                self._register_ingestion_error(CSVIngestionErrors.COURSE_RUN_CREATE_ERROR, exception_message)
+                raise Exception(error_message)  # pylint: disable=raise-missing-from
+        if not course_run and is_course_run_created:
+            course_run = CourseRun.objects.filter_drafts(course=course).order_by('created').last()
+        return course_run, is_course_run_created
+
     def validate_course_data(self, course_type, data):
         """
         Verify the required data key-values for a course type are present in the provided
@@ -339,17 +401,24 @@ def get_ingestion_stats(self):
         }
 
     def _register_successful_ingestion(
-            self, course_uuid, created, active_url_slug, external_course_marketing_type=None):
+        self,
+        course_uuid,
+        is_course_created,
+        is_course_run_created,
+        active_url_slug,
+        external_course_marketing_type=None
+    ):
         """
         Register the summary of a successful ingestion.
         """
         self.ingestion_summary['success_count'] += 1
-        if created:
+        if is_course_created or is_course_run_created:
             self.ingestion_summary['created_products'].append(
                 {
                     'uuid': course_uuid,
                     'external_course_marketing_type': external_course_marketing_type,
-                    'url_slug': active_url_slug
+                    'url_slug': active_url_slug,
+                    're-run': is_course_run_created
                 }
             )
         else:
@@ -370,6 +439,7 @@ def _create_course_api_request_data(self, data, course_type, course_run_type_uui
             'run_type': str(course_run_type_uuid),
             'prices': pricing,
         }
+
         return {
             'org': data['organization'],
             'title': data['title'],
@@ -380,15 +450,33 @@ def _create_course_api_request_data(self, data, course_type, course_run_type_uui
             'course_run': course_run_creation_fields
         }
 
-    def get_draft_flag(self, course_run):
+    def _create_course_run_api_request_data(self, data, course, course_type, course_run_type_uuid, rerun=None):
+        """
+        Given a data dictionary, return a reduced data representation in dict
+        which will be used as input for course run creation via course run api.
+        """
+        pricing = self.get_pricing_representation(data['verified_price'], course_type)
+        course_run_creation_fields = {
+            'pacing_type': self.get_pacing_type(data['course_pacing']),
+            'start': self.get_formatted_datetime_string(f"{data['start_date']} {data['start_time']}"),
+            'end': self.get_formatted_datetime_string(f"{data['end_date']} {data['end_time']}"),
+            'run_type': str(course_run_type_uuid),
+            'prices': pricing,
+            'course': course.key,
+        }
+        if rerun:
+            course_run_creation_fields['rerun'] = rerun
+        return course_run_creation_fields
+
+    def get_draft_flag(self, course):
         """
         To keep behavior of CSV loader consistent with publisher, draft flag is false only when:
             1. Course run is moved from Unpublished -> Review State
             2. Any of the Course run is in published state
-        No 1 is not applicable at the moment. For 2, CSV loader right now only expects
-        one course run for each course, hence the status of the single fetched course run is checked.
+        No 1 is not applicable at the moment as we are changing status via CSV loader, so we are sending false
+        draft flag to the course_run api directly for now.
         """
-        return not course_run.status == CourseRunStatus.Published
+        return not CourseRun.objects.filter_drafts(course=course, status=CourseRunStatus.Published).exists()
 
     def _archive_stale_products(self, course_external_identifiers):
         """
@@ -462,6 +550,8 @@ def _update_course_run_request_data(self, data, course_run, course_type, is_draf
         staff_uuids = self.process_staff_names(data.get('staff', ''), course_run.key)
         content_language = self.verify_and_get_language_tags(data['content_language'])
         transcript_language = self.verify_and_get_language_tags(data['transcript_language'])
+        registration_deadline = data.get('reg_close_date', '')
+        variant_id = data.get('variant_id', '')
 
         update_course_run_data = {
             'run_type': str(course_run.type.uuid),
@@ -483,6 +573,13 @@ def _update_course_run_request_data(self, data, course_run, course_type, is_draf
                 f"{data.get('upgrade_deadline_override_time', '')}".strip()
             ),
         }
+
+        if registration_deadline:
+            update_course_run_data.update({'enrollment_end': self.get_formatted_datetime_string(
+                f"{data['reg_close_date']} {data['reg_close_time']}"
+            )})
+        if variant_id:
+            update_course_run_data.update({'variant_id': variant_id})
         return update_course_run_data
 
     def get_formatted_datetime_string(self, date_string):
@@ -554,6 +651,17 @@ def _create_course(self, data, course_type, course_run_type_uuid):
             logger.info("Course creation response: {}".format(response.content))  # lint-amnesty, pylint: disable=logging-format-interpolation
         return response.json()
 
+    def _create_course_run(self, data, course, course_type, course_run_type_uuid, rerun=None):
+        """
+        Make a course run entry through course run api.
+        """
+        url = f"{settings.DISCOVERY_BASE_URL}{reverse('api:v1:course_run-list')}"
+        request_data = self._create_course_run_api_request_data(data, course, course_type, course_run_type_uuid, rerun)
+        response = self._call_course_api('POST', url, request_data)
+        if response.status_code not in (200, 201):
+            logger.info("Course run creation response: {}".format(response.content))  # lint-amnesty, pylint: disable=logging-format-interpolation
+        return response.json()
+
     def _update_course(self, data, course, is_draft):
         """
         Update the course data.

course_discovery/apps/course_metadata/data_loaders/tests/mixins.py

@@ -368,8 +368,8 @@ class CSVLoaderMixin:
         },
         'facts_data': ['90 million', '<p>Bacterias cottage cost</p>', 'Diamond mine', '<p>Worth it</p>'],
         'start_date': '2020-01-25T00:00:00+00:00',
-        'end_date': '2020-02-25T00:00:00+00:00',
-        'registration_deadline': '2020-01-25T00:00:00+00:00',
+        'end_date': '2050-02-25T00:00:00+00:00',
+        'registration_deadline': '2050-01-25T00:00:00+00:00',
         'variant_id': "00000000-0000-0000-0000-000000000000",
         "meta_title": "SEO Title",
         "meta_description": "SEO Description",
@@ -391,6 +391,7 @@ class CSVLoaderMixin:
         'go_live_date': '2020-01-25T00:00:00+00:00',
         'expected_program_type': 'professional-certificate',
         'expected_program_name': 'New Program for all',
+        'registration_deadline': '2050-01-25T00:00:00+00:00',
     }
 
     def setUp(self):
@@ -495,7 +496,6 @@ def _assert_course_data(self, course, expected_data):
         course_entitlement = CourseEntitlement.everything.get(
             draft=expected_data['draft'], mode__slug=self.paid_exec_ed_slug, course=course
         )
-
         assert course.draft is expected_data['draft']
         assert course.title == expected_data['title']
         assert course.faq == expected_data['faq']
@@ -547,6 +547,7 @@ def _assert_course_run_data(self, course_run, expected_data):
         assert course_run_seat.draft is expected_data['draft']
         assert course_run.status == expected_data['status']
         assert course_run.weeks_to_complete == expected_data['length']
+        assert course_run.enrollment_end.isoformat() == expected_data['registration_deadline']
         assert course_run.min_effort == expected_data['minimum_effort']
         assert course_run.max_effort == expected_data['maximum_effort']
         assert course_run_seat.price == expected_data['verified_price']

course_discovery/apps/course_metadata/data_loaders/tests/mock_data.py

@@ -3200,9 +3200,9 @@
     'publish_date': '01/25/2020',
     'start_date': '01/25/2020',
     'start_time': '00:00',
-    'end_date': '02/25/2020',
+    'end_date': '02/25/2050',
     'end_time': '00:00',
-    'reg_close_date': '01/25/2020',
+    'reg_close_date': '01/25/2050',
     'reg_close_time': '00:00',
     'course_pacing': 'self-paced',
     'course_run_enrollment_track': 'Paid Executive Education',