Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[COST-5135] Update Masu endpoint to handle new ocp on cloud flow #5277

Merged
merged 6 commits into from
Nov 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion koku/masu/api/process_openshift_on_cloud.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@
from common.queues import DownloadQueue
from common.queues import get_customer_queue
from common.queues import QUEUE_LIST
from common.queues import SummaryQueue
from masu.processor import is_managed_ocp_cloud_processing_enabled
from masu.processor.tasks import process_daily_openshift_on_cloud as process_daily_openshift_on_cloud_task
from masu.processor.tasks import process_openshift_on_cloud as process_openshift_on_cloud_task
from masu.processor.tasks import process_openshift_on_cloud_trino

LOG = logging.getLogger(__name__)
REPORT_DATA_KEY = "process_openshift_on_cloud Task IDs"
Expand Down Expand Up @@ -68,7 +71,33 @@ def process_openshift_on_cloud(request):
errmsg = f"You must provide a cloud provider UUID from {Provider.OPENSHIFT_ON_CLOUD_PROVIDER_LIST}."
return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST)

months = get_months_in_date_range(start=start_date, end=end_date)
if provider.type in [Provider.PROVIDER_GCP, Provider.PROVIDER_GCP_LOCAL]:
invoice_month = DateHelper().invoice_month_from_bill_date(start_date)
months = get_months_in_date_range(start=start_date, end=end_date, invoice_month=invoice_month)
else:
months = get_months_in_date_range(start=start_date, end=end_date)

if provider.type in Provider.MANAGED_OPENSHIFT_ON_CLOUD_PROVIDER_LIST and is_managed_ocp_cloud_processing_enabled(
schema_name
):
fallback_queue = get_customer_queue(schema_name, SummaryQueue)
for month in months:
tracing_id = str(uuid4())
report = {
"schema_name": schema_name,
"provider_type": provider.type,
"provider_uuid": cloud_provider_uuid,
"tracing_id": tracing_id,
"start": month[0],
"end": month[1],
"invoice_month": month[2],
}
ocp_async = process_openshift_on_cloud_trino.s(
[report], provider.type, schema_name, cloud_provider_uuid, tracing_id, masu_api_trigger=True
).apply_async(queue=queue_name or fallback_queue)
async_results.append({f"Managed OCP on Cloud {str(month)}": str(ocp_async)})
return Response({REPORT_DATA_KEY: async_results})

bill_dates = [ciso8601.parse_datetime(month[0]).replace(day=1).strftime("%Y-%m-%d") for month in months]

for bill_date in bill_dates:
Expand Down
2 changes: 1 addition & 1 deletion koku/masu/database/aws_report_db_accessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params):
def populate_ocp_on_cloud_daily_trino(
self, aws_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags
):
"""Populate the aws_openshift_daily trino table for OCP on AWS.
"""Populate the managed_aws_openshift_daily trino table for OCP on AWS.
Args:
aws_provider_uuid (UUID) AWS source UUID.
ocp_provider_uuid (UUID) OCP source UUID.
Expand Down
2 changes: 1 addition & 1 deletion koku/masu/database/azure_report_db_accessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params):
def populate_ocp_on_cloud_daily_trino(
self, azure_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags
):
"""Populate the azure_openshift_daily trino table for OCP on Azure.
"""Populate the managed_azure_openshift_daily trino table for OCP on Azure.
Args:
azure_provider_uuid (UUID) GCP source UUID.
ocp_provider_uuid (UUID) OCP source UUID.
Expand Down
2 changes: 1 addition & 1 deletion koku/masu/database/gcp_report_db_accessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params):
def populate_ocp_on_cloud_daily_trino(
self, gcp_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags
):
"""Populate the gcp_openshift_daily trino table for OCP on Azure.
"""Populate the managed_gcp_openshift_daily trino table for OCP on GCP.
Args:
gcp_provider_uuid (UUID) GCP source UUID.
ocp_provider_uuid (UUID) OCP source UUID.
Expand Down
4 changes: 2 additions & 2 deletions koku/masu/processor/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1255,7 +1255,7 @@ def validate_daily_data(schema, start_date, end_date, provider_uuid, ocp_on_clou

@celery_app.task(name="masu.processor.tasks.process_openshift_on_cloud_trino", queue=SummaryQueue.DEFAULT, bind=True)
def process_openshift_on_cloud_trino(
self, reports_to_summarize, provider_type, schema_name, provider_uuid, tracing_id
self, reports_to_summarize, provider_type, schema_name, provider_uuid, tracing_id, masu_api_trigger=False
):
"""Process OCP on Cloud data into managed tables for summary"""
reports_deduplicated = deduplicate_summary_reports(reports_to_summarize, manifest_list=[])
Expand All @@ -1275,7 +1275,7 @@ def process_openshift_on_cloud_trino(
with ReportManifestDBAccessor() as manifest_accesor:
tracing_id = report.get("tracing_id", report.get("manifest_uuid", "no-tracing-id"))

if not manifest_accesor.manifest_ready_for_summary(report.get("manifest_id")):
if not masu_api_trigger and not manifest_accesor.manifest_ready_for_summary(report.get("manifest_id")):
LOG.info(log_json(tracing_id, msg="manifest not ready for summary", context=report))
continue

Expand Down
24 changes: 24 additions & 0 deletions koku/masu/test/api/test_process_openshift_on_cloud.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,27 @@ def test_get_process_daily_openshift_on_cloud(self, mock_process, _):
self.assertEqual(response.status_code, 200)
self.assertIn(expected_key, body)
mock_process.s.return_value.apply_async.assert_called()

@patch("masu.api.process_openshift_on_cloud.process_openshift_on_cloud_trino")
@patch("masu.api.process_openshift_on_cloud.is_managed_ocp_cloud_processing_enabled")
@patch("koku.middleware.MASU")
def test_get_process_daily_openshift_on_cloud_trino(self, middleware, unleash, mock_task):
"""Test the GET report_data endpoint for a provider that is daily partitioned."""
unleash.return_value = True
middleware.return_value = True
dh = DateHelper()
start_date = str(dh.this_month_start.date())
end_date = str(dh.this_month_end.date())
params = {
"schema": self.schema,
"provider_uuid": self.gcp_provider_uuid,
"start_date": start_date,
"end_date": end_date,
}
expected_key = "process_openshift_on_cloud Task IDs"
response = self.client.get(reverse("process_openshift_on_cloud"), params)
body = response.json()

self.assertEqual(response.status_code, 200)
self.assertIn(expected_key, body)
mock_task.s.return_value.apply_async.assert_called()