From 8e5a760177345e5439ac7dae71dd756e05937bc1 Mon Sep 17 00:00:00 2001 From: Phillip Jensen <phillip@golemgrid.com> Date: Thu, 19 Sep 2024 11:29:54 +0200 Subject: [PATCH] Use only relay to index online status, and update nodestatushistory to include any type of node rather than linked to provider --- .../golem-reputation-backend/p2p-ping.py | 5 + .../0052_update_nodestatushistory.py | 42 ++++++ ...tory_idx_api_nodesta_node_id_acbc40_idx.py | 18 +++ .../reputation-backend/api/models.py | 6 +- .../reputation-backend/api/ping.py | 12 +- .../reputation-backend/api/scanner.py | 142 ++++++------------ .../reputation-backend/api/scoring.py | 2 +- .../reputation-backend/api/tasks.py | 78 +++++++++- .../reputation-backend/api/utils.py | 40 ++++- .../reputation-backend/api2/api.py | 6 +- .../reputation-backend/core/celery.py | 8 +- .../reputation-backend/stats/tasks.py | 67 +++++---- 12 files changed, 283 insertions(+), 143 deletions(-) create mode 100644 docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0052_update_nodestatushistory.py create mode 100644 docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0053_rename_node_status_history_idx_api_nodesta_node_id_acbc40_idx.py diff --git a/docker-stack/golem-reputation-backend/p2p-ping.py b/docker-stack/golem-reputation-backend/p2p-ping.py index fd78cfc..ae65b4f 100644 --- a/docker-stack/golem-reputation-backend/p2p-ping.py +++ b/docker-stack/golem-reputation-backend/p2p-ping.py @@ -106,6 +106,10 @@ async def ping_provider(provider_id): results.append(result) else: print("ERROR pinging", stderr.decode()) + # If you detect the critical error related to `yagna.sock`, exit the script + if "No such file or directory" in stderr.decode(): + print("Critical error: yagna.sock is unavailable, exiting...") + os._exit(1) # This will exit the script and stop the container return False if len(results) == 2: @@ -129,6 +133,7 @@ async def ping_provider(provider_id): print("Timeout reached while checking node status", e) return False + # Process each provider ID and send ping results in chunks diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0052_update_nodestatushistory.py b/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0052_update_nodestatushistory.py new file mode 100644 index 0000000..4122ae7 --- /dev/null +++ b/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0052_update_nodestatushistory.py @@ -0,0 +1,42 @@ +from django.db import migrations, models + +def update_nodestatushistory(apps, schema_editor): + NodeStatusHistory = apps.get_model('api', 'NodeStatusHistory') + Provider = apps.get_model('api', 'Provider') + + for history in NodeStatusHistory.objects.all(): + if history.provider: + history.node_id = history.provider.node_id + history.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0051_update_gputask_structure'), + ] + + operations = [ + migrations.AddField( + model_name='nodestatushistory', + name='node_id', + field=models.CharField(max_length=42, null=True), + ), + migrations.RunPython(update_nodestatushistory), + migrations.RemoveField( + model_name='nodestatushistory', + name='provider', + ), + migrations.AlterField( + model_name='nodestatushistory', + name='node_id', + field=models.CharField(max_length=42), + ), + migrations.RemoveIndex( + model_name='nodestatushistory', + name='api_nodesta_provide_94f647_idx', + ), + migrations.AddIndex( + model_name='nodestatushistory', + index=models.Index(fields=['node_id', 'timestamp'], name='node_status_history_idx'), + ), + ] \ No newline at end of file diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0053_rename_node_status_history_idx_api_nodesta_node_id_acbc40_idx.py b/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0053_rename_node_status_history_idx_api_nodesta_node_id_acbc40_idx.py new file mode 100644 index 0000000..49ec403 --- /dev/null +++ b/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0053_rename_node_status_history_idx_api_nodesta_node_id_acbc40_idx.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.7 on 2024-09-19 10:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0052_update_nodestatushistory'), + ] + + operations = [ + migrations.RenameIndex( + model_name='nodestatushistory', + new_name='api_nodesta_node_id_acbc40_idx', + old_name='node_status_history_idx', + ), + ] diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/models.py b/docker-stack/golem-reputation-backend/reputation-backend/api/models.py index 06cd874..91f98d7 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/api/models.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/api/models.py @@ -263,14 +263,14 @@ class Meta: class NodeStatusHistory(models.Model): - provider = models.ForeignKey(Provider, on_delete=models.CASCADE) + node_id = models.CharField(max_length=42) is_online = models.BooleanField() timestamp = models.DateTimeField(auto_now_add=True) def __str__(self): - return f"{self.provider.node_id} - {'Online' if self.is_online else 'Offline'} at {self.timestamp}" + return f"{self.node_id} - {'Online' if self.is_online else 'Offline'} at {self.timestamp}" class Meta: indexes = [ - models.Index(fields=["provider", "timestamp"]), + models.Index(fields=["node_id", "timestamp"]), ] diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/ping.py b/docker-stack/golem-reputation-backend/reputation-backend/api/ping.py index d16d6f6..f47bdc4 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/api/ping.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/api/ping.py @@ -10,14 +10,14 @@ async def async_fetch_node_ids(): # Define the synchronous part as an inner function def get_node_ids(): - # Fetch the latest status for each provider and filter those that are online + # Fetch the latest status for each node_id and filter those that are online latest_statuses = NodeStatusHistory.objects.filter( - provider_id__in=NodeStatusHistory.objects.order_by( - 'provider', '-timestamp').distinct('provider').values_list('provider_id', flat=True) - ).order_by('provider', '-timestamp').distinct('provider') + node_id__in=NodeStatusHistory.objects.order_by( + 'node_id', '-timestamp').distinct('node_id').values_list('node_id', flat=True) + ).order_by('node_id', '-timestamp').distinct('node_id') - # Return provider IDs where the latest status is online - return [status.provider.node_id for status in latest_statuses if status.is_online] + # Return node_ids where the latest status is online + return [status.node_id for status in latest_statuses if status.is_online] # Use sync_to_async to convert it and immediately invoke node_ids = await sync_to_async(get_node_ids, thread_sensitive=True)() diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/scanner.py b/docker-stack/golem-reputation-backend/reputation-backend/api/scanner.py index 61d99c5..1896683 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/api/scanner.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/api/scanner.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import redis +import requests import asyncio import csv import json @@ -24,18 +25,12 @@ @app.task(queue='default', options={'queue': 'default', 'routing_key': 'default'}) def update_providers_info(node_props): - node_ids = [prop['node_id'] for prop in node_props] - existing_providers = Provider.objects.filter(node_id__in=node_ids) - existing_providers_dict = { - provider.node_id: provider for provider in existing_providers} - - create_batch = [] - update_batch = [] - + provider_data = [] for props in node_props: prop_data = {key: value for key, value in props.items() if key.startswith( "golem.com.payment.platform.") and key.endswith(".address")} - provider_data = { + provider_data.append({ + "node_id": props['node_id'], "payment_addresses": prop_data, "network": 'testnet' if any(key in TESTNET_KEYS for key in props.keys()) else 'mainnet', "cores": props.get("golem.inf.cpu.cores"), @@ -46,20 +41,32 @@ def update_providers_info(node_props): "threads": props.get("golem.inf.cpu.threads"), "storage": props.get("golem.inf.storage.gib"), "name": props.get("golem.node.id.name"), - } - - issuer_id = props['node_id'] - if issuer_id in existing_providers_dict: - provider_instance = existing_providers_dict[issuer_id] - for key, value in provider_data.items(): - setattr(provider_instance, key, value) - update_batch.append(provider_instance) + }) + + node_ids = [data['node_id'] for data in provider_data] + existing_providers = { + provider.node_id: provider + for provider in Provider.objects.filter(node_id__in=node_ids) + } + + providers_to_create = [] + providers_to_update = [] + + for data in provider_data: + if data['node_id'] in existing_providers: + provider = existing_providers[data['node_id']] + for key, value in data.items(): + setattr(provider, key, value) + providers_to_update.append(provider) else: - create_batch.append(Provider(node_id=issuer_id, **provider_data)) + providers_to_create.append(Provider(**data)) - Provider.objects.bulk_create(create_batch, ignore_conflicts=True) + Provider.objects.bulk_create(providers_to_create, ignore_conflicts=True) Provider.objects.bulk_update( - update_batch, [field for field in provider_data.keys() if field != 'node_id']) + providers_to_update, + fields=[field for field in provider_data[0].keys() if field != + 'node_id'] + ) TESTNET_KEYS = [ @@ -76,81 +83,33 @@ def update_providers_info(node_props): from .utils import build_parser, print_env_info, format_usage # noqa: E402 -def update_nodes_status(provider_id, is_online_now): - provider, created = Provider.objects.get_or_create(node_id=provider_id) - - # Get the latest status from Redis - latest_status = r.get(f"provider:{provider_id}:status") - - if latest_status is None: - # Status not found in Redis, fetch the latest status from the database - last_status = NodeStatusHistory.objects.filter( - provider=provider).last() - if not last_status or last_status.is_online != is_online_now: - # Create a new status entry if there's a change in status - NodeStatusHistory.objects.create( - provider=provider, is_online=is_online_now) - provider.online = is_online_now - provider.save() - else: - # Compare the latest status from Redis with the current status - if latest_status.decode() != str(is_online_now): - # Status has changed, update the database and Node.online field - NodeStatusHistory.objects.create( - provider=provider, is_online=is_online_now) - provider.online = is_online_now - provider.save() - - # Store the current status in Redis for future lookups - r.set(f"provider:{provider_id}:status", str(is_online_now)) - - -@app.task(queue='uptime', options={'queue': 'uptime', 'routing_key': 'uptime'}) -def update_nodes_data(node_props): - r = redis.Redis(host='redis', port=6379, db=0) - - for props in node_props: - issuer_id = props['node_id'] - is_online_now = check_node_status(issuer_id) - print(f"Updating NodeStatus for {issuer_id} with is_online_now={is_online_now}") - try: - update_nodes_status(issuer_id, is_online_now) - except Exception as e: - print(f"Error updating NodeStatus for {issuer_id}: {e}") - - provider_ids_in_props = {props['node_id'] for props in node_props} - previously_online_providers_ids = Provider.objects.filter( - nodestatushistory__is_online=True - ).distinct().values_list('node_id', flat=True) - - provider_ids_not_in_scan = set( - previously_online_providers_ids) - provider_ids_in_props - - for issuer_id in provider_ids_not_in_scan: - is_online_now = check_node_status(issuer_id) - print(f"Verifying NodeStatus for {issuer_id} with is_online_now={is_online_now}") - try: - update_nodes_status(issuer_id, is_online_now) - except Exception as e: - print(f"Error verifying/updating NodeStatus for {issuer_id}: {e}") - - def check_node_status(issuer_id): + node_id_no_prefix = issuer_id[2:] if issuer_id.startswith( + '0x') else issuer_id + url = f"http://yacn2.dev.golem.network:9000/nodes/{node_id_no_prefix}" try: - process = subprocess.run( - ["yagna", "net", "find", issuer_id], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=5 # 5-second timeout for the subprocess - ) - - # Process finished, return True if it was successful and "seen:" is in the output - return process.returncode == 0 and "seen:" in process.stdout.decode() - except subprocess.TimeoutExpired as e: - print("Timeout reached while checking node status", e) + response = requests.get(url, timeout=5) + response.raise_for_status() + data = response.json() + node_key = issuer_id.lower() + node_info = data.get(node_key) + + if node_info: + if isinstance(node_info, list): + if node_info == [] or node_info == [None]: + return False + else: + return any('seen' in item for item in node_info if item) + else: + return False + else: + return False + except requests.exceptions.RequestException as e: + print( + f"HTTP request exception when checking node status for {issuer_id}: {e}") return False except Exception as e: - print(f"Unexpected error checking node status: {e}") + print(f"Unexpected error checking node status for {issuer_id}: {e}") return False @@ -200,5 +159,4 @@ async def monitor_nodes_status(subnet_tag: str = "public"): print("Scan timeout reached") # Delay update_nodes_data call using Celery - update_nodes_data.delay(node_props) update_providers_info.delay(node_props) diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/scoring.py b/docker-stack/golem-reputation-backend/reputation-backend/api/scoring.py index 8456460..fe1fe56 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/api/scoring.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/api/scoring.py @@ -25,7 +25,7 @@ def penalty_weight(deviation): def calculate_uptime(node_id, node=None): if node is None: node = Provider.objects.get(node_id=node_id) - statuses = NodeStatusHistory.objects.filter(provider=node).order_by("timestamp") + statuses = NodeStatusHistory.objects.filter(node_id=node_id).order_by("timestamp") online_duration = timedelta(0) last_online_time = None diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/tasks.py b/docker-stack/golem-reputation-backend/reputation-backend/api/tasks.py index ea73983..f9233c6 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/api/tasks.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/api/tasks.py @@ -70,9 +70,8 @@ def update_provider_scores(network): r = redis.Redis(host='redis', port=6379, db=0) ten_days_ago = timezone.now() - timedelta(days=10) recent_online_providers = NodeStatusHistory.objects.filter( - is_online=True).order_by('provider', '-timestamp').distinct('provider') - online_provider_ids = [ - status.provider_id for status in recent_online_providers] + is_online=True).order_by('node_id', '-timestamp').distinct('node_id') + online_provider_ids = [status.node_id for status in recent_online_providers] providers = Provider.objects.filter(node_id__in=online_provider_ids, network=network).annotate( success_count=Count('taskcompletion', filter=Q( taskcompletion__is_successful=True, taskcompletion__timestamp__gte=ten_days_ago)), @@ -241,7 +240,7 @@ def get_blacklisted_operators(): latest=Max('timestamp', filter=Q(is_online=True)) ).filter( timestamp=F('latest'), is_online=True - ).values_list('provider_id', flat=True) + ).values_list('node_id', flat=True) providers = Provider.objects.filter( node_id__in=latest_online_statuses @@ -406,4 +405,73 @@ def delete_old_ping_results(): # Optionally, you can log the number of deleted records or return it print( - f"Deleted {count_ping_results} PingResult records older than 30 days.") \ No newline at end of file + f"Deleted {count_ping_results} PingResult records older than 30 days.") + +import requests +from .utils import check_node_status +from django.db import transaction +r = redis.Redis(host='redis', port=6379, db=0) + +@app.task +def fetch_and_update_relay_nodes_online_status(): + base_url = "http://yacn2.dev.golem.network:9000/nodes/" + current_online_nodes = set() + nodes_to_update = [] + + for prefix in range(256): + try: + response = requests.get(f"{base_url}{prefix:02x}", timeout=5) + response.raise_for_status() + data = response.json() + + for node_id, sessions in data.items(): + node_id = node_id.strip().lower() + is_online = bool(sessions) and any('seen' in item for item in sessions if item) + current_online_nodes.add(node_id) + nodes_to_update.append((node_id, is_online)) + + except requests.RequestException as e: + print(f"Error fetching data for prefix {prefix:02x}: {e}") + + # Bulk update node statuses + bulk_update_node_statuses.delay(nodes_to_update) + + # Check providers that were previously online but not found in the current scan + previously_online = set(NodeStatusHistory.objects.filter( + is_online=True + ).order_by('node_id', '-timestamp').distinct('node_id').values_list('node_id', flat=True)) + + missing_nodes = previously_online - current_online_nodes + if missing_nodes: + check_missing_nodes.delay(list(missing_nodes)) + +@app.task +def bulk_update_node_statuses(nodes_data): + status_history_to_create = [] + redis_updates = {} + + for node_id, is_online in nodes_data: + latest_status = r.get(f"provider:{node_id}:status") + + if latest_status is None or latest_status.decode() != str(is_online): + status_history_to_create.append( + NodeStatusHistory(node_id=node_id, is_online=is_online) + ) + redis_updates[f"provider:{node_id}:status"] = str(is_online) + + + if status_history_to_create: + with transaction.atomic(): + NodeStatusHistory.objects.bulk_create(status_history_to_create) + + if redis_updates: + r.mset(redis_updates) + +@app.task +def check_missing_nodes(missing_nodes): + nodes_to_update = [] + for node_id in missing_nodes: + is_online = check_node_status(node_id) + nodes_to_update.append((node_id, is_online)) + + bulk_update_node_statuses(nodes_to_update) diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/utils.py b/docker-stack/golem-reputation-backend/reputation-backend/api/utils.py index 9246928..47033d9 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/api/utils.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/api/utils.py @@ -105,4 +105,42 @@ def run_golem_example(example_main, log_file=None): f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}" ) except (asyncio.CancelledError, KeyboardInterrupt): - pass \ No newline at end of file + pass + + +import requests +def check_node_status(node_id): + node_id_no_prefix = node_id[2:] if node_id.startswith('0x') else node_id + url = f"http://yacn2.dev.golem.network:9000/nodes/{node_id_no_prefix}" + try: + response = requests.get(url, timeout=5) + response.raise_for_status() + data = response.json() + + # If the response is an empty dictionary, the node is considered offline + if not data: + return False + + node_key = node_id.lower() + node_info = data.get(node_key) + + if node_info: + if isinstance(node_info, list): + if node_info == [] or node_info == [None]: + return False + else: + return any('seen' in item for item in node_info if item) + else: + return False + else: + return False + except requests.exceptions.RequestException as e: + print(url, node_id) + # Log the error and return False + print(f"HTTP request exception when checking node status for {node_id}: {e}") + return False + except Exception as e: + # Log the error and return False + print(url, node_id) + print(f"Unexpected error checking node status for {node_id}: {e}") + return False \ No newline at end of file diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api2/api.py b/docker-stack/golem-reputation-backend/reputation-backend/api2/api.py index 0f6bd9a..371dd42 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/api2/api.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/api2/api.py @@ -176,7 +176,7 @@ def filter_providers( ).annotate( latest_status=Subquery( NodeStatusHistory.objects.filter( - provider=OuterRef('pk') + node_id=OuterRef('node_id') ).order_by('-timestamp').values('is_online')[:1] ) ).filter(latest_status=True) @@ -868,7 +868,7 @@ def list_all_provider_scores(request): def get_score_overview(request): # Filter providers whose latest NodeStatusHistory is_online=True latest_status_subquery = NodeStatusHistory.objects.filter( - provider=OuterRef('pk') + node_id=OuterRef('node_id') ).order_by('-timestamp').values('is_online')[:1] providers = Provider.objects.annotate( @@ -878,7 +878,7 @@ def get_score_overview(request): # Calculate uptime for each provider def calculate_uptime(provider): statuses = NodeStatusHistory.objects.filter( - provider=provider).order_by('timestamp') + node_id=provider.node_id).order_by('timestamp') online_duration = timedelta(0) last_online_time = None diff --git a/docker-stack/golem-reputation-backend/reputation-backend/core/celery.py b/docker-stack/golem-reputation-backend/reputation-backend/core/celery.py index 31f59ee..bf01c8e 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/core/celery.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/core/celery.py @@ -15,8 +15,14 @@ @app.on_after_finalize.connect def setup_periodic_tasks(sender, **kwargs): - from api.tasks import monitor_nodes_task, ping_providers_task, process_offers_from_redis, update_provider_scores, get_blacklisted_operators, get_blacklisted_providers, delete_old_ping_results + from api.tasks import monitor_nodes_task, ping_providers_task, process_offers_from_redis, update_provider_scores, get_blacklisted_operators, get_blacklisted_providers, delete_old_ping_results, fetch_and_update_relay_nodes_online_status from stats.tasks import populate_daily_provider_stats, cache_provider_success_ratio, cache_provider_uptime, cache_cpu_performance_ranking, cache_gpu_performance_ranking + sender.add_periodic_task( + 30.0, + fetch_and_update_relay_nodes_online_status.s(), + queue="default", + options={"queue": "default", "routing_key": "default"}, + ) sender.add_periodic_task( crontab(minute=0, hour=0), # daily at midnight cache_provider_success_ratio.s(), diff --git a/docker-stack/golem-reputation-backend/reputation-backend/stats/tasks.py b/docker-stack/golem-reputation-backend/reputation-backend/stats/tasks.py index edfc331..9105839 100644 --- a/docker-stack/golem-reputation-backend/reputation-backend/stats/tasks.py +++ b/docker-stack/golem-reputation-backend/reputation-backend/stats/tasks.py @@ -70,18 +70,18 @@ def populate_daily_provider_stats(): @app.task def cache_provider_uptime(): - # Get the latest online status for each provider latest_statuses = NodeStatusHistory.objects.filter( timestamp=Subquery( - NodeStatusHistory.objects.filter(provider=OuterRef('provider')) + NodeStatusHistory.objects.filter(node_id=OuterRef('node_id')) .order_by('-timestamp') .values('timestamp')[:1] ) ) - provider_ids = [status.provider_id for status in latest_statuses if status.is_online] - + # Get online node_ids that also exist in the Provider model + online_node_ids = [status.node_id for status in latest_statuses if status.is_online] + existing_providers = Provider.objects.filter(node_id__in=online_node_ids).values_list('node_id', flat=True) uptime_data = { '100-90': 0, @@ -92,20 +92,23 @@ def cache_provider_uptime(): '20-0': 0 } - for provider_id in provider_ids: - uptime_percentage = calculate_uptime(provider_id) - if uptime_percentage >= 90: - uptime_data['100-90'] += 1 - elif uptime_percentage >= 80: - uptime_data['90-80'] += 1 - elif uptime_percentage >= 60: - uptime_data['80-60'] += 1 - elif uptime_percentage >= 40: - uptime_data['60-40'] += 1 - elif uptime_percentage >= 20: - uptime_data['40-20'] += 1 - else: - uptime_data['20-0'] += 1 + for provider_id in existing_providers: + try: + uptime_percentage = calculate_uptime(provider_id) + if uptime_percentage >= 90: + uptime_data['100-90'] += 1 + elif uptime_percentage >= 80: + uptime_data['90-80'] += 1 + elif uptime_percentage >= 60: + uptime_data['80-60'] += 1 + elif uptime_percentage >= 40: + uptime_data['60-40'] += 1 + elif uptime_percentage >= 20: + uptime_data['40-20'] += 1 + else: + uptime_data['20-0'] += 1 + except Exception as e: + print(f"Error calculating uptime for provider {provider_id}: {str(e)}") redis_client.set('stats_provider_uptime', json.dumps(uptime_data)) @@ -115,14 +118,16 @@ def cache_provider_success_ratio(): # Get the latest online status for each provider latest_statuses = NodeStatusHistory.objects.filter( timestamp=Subquery( - NodeStatusHistory.objects.filter(provider=OuterRef('provider')) + NodeStatusHistory.objects.filter(node_id=OuterRef('node_id')) .order_by('-timestamp') .values('timestamp')[:1] ) ) - provider_ids = [status.provider_id for status in latest_statuses if status.is_online] + online_node_ids = [status.node_id for status in latest_statuses if status.is_online] + # Get providers that exist in the Provider model and are online + existing_providers = Provider.objects.filter(node_id__in=online_node_ids) # Calculate success ratios success_ratio_data = { @@ -134,15 +139,12 @@ def cache_provider_success_ratio(): '20-0': 0 } - for provider_id in provider_ids: - provider = Provider.objects.filter(node_id=provider_id).annotate( - success_count=Count('taskcompletion', filter=Q( - taskcompletion__is_successful=True)), - total_count=Count('taskcompletion'), - ).first() - - if provider and provider.total_count > 0: - success_ratio = provider.success_count / provider.total_count * 100 + for provider in existing_providers.annotate( + success_count=Count('taskcompletion', filter=Q(taskcompletion__is_successful=True)), + total_count=Count('taskcompletion') + ): + if provider.total_count > 0: + success_ratio = (provider.success_count / provider.total_count) * 100 if success_ratio >= 90: success_ratio_data['100-90'] += 1 elif success_ratio >= 80: @@ -156,8 +158,11 @@ def cache_provider_success_ratio(): else: success_ratio_data['20-0'] += 1 - redis_client.set('stats_provider_success_ratio', - json.dumps(success_ratio_data)) + redis_client.set('stats_provider_success_ratio', json.dumps(success_ratio_data)) + + + + from django.db.models import Max, F, Subquery, OuterRef, Prefetch