From 8e5a760177345e5439ac7dae71dd756e05937bc1 Mon Sep 17 00:00:00 2001
From: Phillip Jensen <phillip@golemgrid.com>
Date: Thu, 19 Sep 2024 11:29:54 +0200
Subject: [PATCH] Use only relay to index online status, and update
 nodestatushistory to include any type of node rather than linked to provider

---
 .../golem-reputation-backend/p2p-ping.py      |   5 +
 .../0052_update_nodestatushistory.py          |  42 ++++++
 ...tory_idx_api_nodesta_node_id_acbc40_idx.py |  18 +++
 .../reputation-backend/api/models.py          |   6 +-
 .../reputation-backend/api/ping.py            |  12 +-
 .../reputation-backend/api/scanner.py         | 142 ++++++------------
 .../reputation-backend/api/scoring.py         |   2 +-
 .../reputation-backend/api/tasks.py           |  78 +++++++++-
 .../reputation-backend/api/utils.py           |  40 ++++-
 .../reputation-backend/api2/api.py            |   6 +-
 .../reputation-backend/core/celery.py         |   8 +-
 .../reputation-backend/stats/tasks.py         |  67 +++++----
 12 files changed, 283 insertions(+), 143 deletions(-)
 create mode 100644 docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0052_update_nodestatushistory.py
 create mode 100644 docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0053_rename_node_status_history_idx_api_nodesta_node_id_acbc40_idx.py

diff --git a/docker-stack/golem-reputation-backend/p2p-ping.py b/docker-stack/golem-reputation-backend/p2p-ping.py
index fd78cfc..ae65b4f 100644
--- a/docker-stack/golem-reputation-backend/p2p-ping.py
+++ b/docker-stack/golem-reputation-backend/p2p-ping.py
@@ -106,6 +106,10 @@ async def ping_provider(provider_id):
                 results.append(result)
             else:
                 print("ERROR pinging", stderr.decode())
+                # If you detect the critical error related to `yagna.sock`, exit the script
+                if "No such file or directory" in stderr.decode():
+                    print("Critical error: yagna.sock is unavailable, exiting...")
+                    os._exit(1)  # This will exit the script and stop the container
                 return False
 
         if len(results) == 2:
@@ -129,6 +133,7 @@ async def ping_provider(provider_id):
         print("Timeout reached while checking node status", e)
         return False
 
+
 # Process each provider ID and send ping results in chunks
 
 
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0052_update_nodestatushistory.py b/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0052_update_nodestatushistory.py
new file mode 100644
index 0000000..4122ae7
--- /dev/null
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0052_update_nodestatushistory.py
@@ -0,0 +1,42 @@
+from django.db import migrations, models
+
+def update_nodestatushistory(apps, schema_editor):
+    NodeStatusHistory = apps.get_model('api', 'NodeStatusHistory')
+    Provider = apps.get_model('api', 'Provider')
+
+    for history in NodeStatusHistory.objects.all():
+        if history.provider:
+            history.node_id = history.provider.node_id
+            history.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0051_update_gputask_structure'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='nodestatushistory',
+            name='node_id',
+            field=models.CharField(max_length=42, null=True),
+        ),
+        migrations.RunPython(update_nodestatushistory),
+        migrations.RemoveField(
+            model_name='nodestatushistory',
+            name='provider',
+        ),
+        migrations.AlterField(
+            model_name='nodestatushistory',
+            name='node_id',
+            field=models.CharField(max_length=42),
+        ),
+        migrations.RemoveIndex(
+            model_name='nodestatushistory',
+            name='api_nodesta_provide_94f647_idx',
+        ),
+        migrations.AddIndex(
+            model_name='nodestatushistory',
+            index=models.Index(fields=['node_id', 'timestamp'], name='node_status_history_idx'),
+        ),
+    ]
\ No newline at end of file
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0053_rename_node_status_history_idx_api_nodesta_node_id_acbc40_idx.py b/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0053_rename_node_status_history_idx_api_nodesta_node_id_acbc40_idx.py
new file mode 100644
index 0000000..49ec403
--- /dev/null
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api/migrations/0053_rename_node_status_history_idx_api_nodesta_node_id_acbc40_idx.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2024-09-19 10:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0052_update_nodestatushistory'),
+    ]
+
+    operations = [
+        migrations.RenameIndex(
+            model_name='nodestatushistory',
+            new_name='api_nodesta_node_id_acbc40_idx',
+            old_name='node_status_history_idx',
+        ),
+    ]
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/models.py b/docker-stack/golem-reputation-backend/reputation-backend/api/models.py
index 06cd874..91f98d7 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/api/models.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api/models.py
@@ -263,14 +263,14 @@ class Meta:
 
 
 class NodeStatusHistory(models.Model):
-    provider = models.ForeignKey(Provider, on_delete=models.CASCADE)
+    node_id = models.CharField(max_length=42)
     is_online = models.BooleanField()
     timestamp = models.DateTimeField(auto_now_add=True)
 
     def __str__(self):
-        return f"{self.provider.node_id} - {'Online' if self.is_online else 'Offline'} at {self.timestamp}"
+        return f"{self.node_id} - {'Online' if self.is_online else 'Offline'} at {self.timestamp}"
 
     class Meta:
         indexes = [
-            models.Index(fields=["provider", "timestamp"]),
+            models.Index(fields=["node_id", "timestamp"]),
         ]
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/ping.py b/docker-stack/golem-reputation-backend/reputation-backend/api/ping.py
index d16d6f6..f47bdc4 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/api/ping.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api/ping.py
@@ -10,14 +10,14 @@
 async def async_fetch_node_ids():
     # Define the synchronous part as an inner function
     def get_node_ids():
-        # Fetch the latest status for each provider and filter those that are online
+        # Fetch the latest status for each node_id and filter those that are online
         latest_statuses = NodeStatusHistory.objects.filter(
-            provider_id__in=NodeStatusHistory.objects.order_by(
-                'provider', '-timestamp').distinct('provider').values_list('provider_id', flat=True)
-        ).order_by('provider', '-timestamp').distinct('provider')
+            node_id__in=NodeStatusHistory.objects.order_by(
+                'node_id', '-timestamp').distinct('node_id').values_list('node_id', flat=True)
+        ).order_by('node_id', '-timestamp').distinct('node_id')
 
-        # Return provider IDs where the latest status is online
-        return [status.provider.node_id for status in latest_statuses if status.is_online]
+        # Return node_ids where the latest status is online
+        return [status.node_id for status in latest_statuses if status.is_online]
 
     # Use sync_to_async to convert it and immediately invoke
     node_ids = await sync_to_async(get_node_ids, thread_sensitive=True)()
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/scanner.py b/docker-stack/golem-reputation-backend/reputation-backend/api/scanner.py
index 61d99c5..1896683 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/api/scanner.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api/scanner.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import redis
+import requests
 import asyncio
 import csv
 import json
@@ -24,18 +25,12 @@
 
 @app.task(queue='default', options={'queue': 'default', 'routing_key': 'default'})
 def update_providers_info(node_props):
-    node_ids = [prop['node_id'] for prop in node_props]
-    existing_providers = Provider.objects.filter(node_id__in=node_ids)
-    existing_providers_dict = {
-        provider.node_id: provider for provider in existing_providers}
-
-    create_batch = []
-    update_batch = []
-
+    provider_data = []
     for props in node_props:
         prop_data = {key: value for key, value in props.items() if key.startswith(
             "golem.com.payment.platform.") and key.endswith(".address")}
-        provider_data = {
+        provider_data.append({
+            "node_id": props['node_id'],
             "payment_addresses": prop_data,
             "network": 'testnet' if any(key in TESTNET_KEYS for key in props.keys()) else 'mainnet',
             "cores": props.get("golem.inf.cpu.cores"),
@@ -46,20 +41,32 @@ def update_providers_info(node_props):
             "threads": props.get("golem.inf.cpu.threads"),
             "storage": props.get("golem.inf.storage.gib"),
             "name": props.get("golem.node.id.name"),
-        }
-
-        issuer_id = props['node_id']
-        if issuer_id in existing_providers_dict:
-            provider_instance = existing_providers_dict[issuer_id]
-            for key, value in provider_data.items():
-                setattr(provider_instance, key, value)
-            update_batch.append(provider_instance)
+        })
+
+    node_ids = [data['node_id'] for data in provider_data]
+    existing_providers = {
+        provider.node_id: provider
+        for provider in Provider.objects.filter(node_id__in=node_ids)
+    }
+
+    providers_to_create = []
+    providers_to_update = []
+
+    for data in provider_data:
+        if data['node_id'] in existing_providers:
+            provider = existing_providers[data['node_id']]
+            for key, value in data.items():
+                setattr(provider, key, value)
+            providers_to_update.append(provider)
         else:
-            create_batch.append(Provider(node_id=issuer_id, **provider_data))
+            providers_to_create.append(Provider(**data))
 
-    Provider.objects.bulk_create(create_batch, ignore_conflicts=True)
+    Provider.objects.bulk_create(providers_to_create, ignore_conflicts=True)
     Provider.objects.bulk_update(
-        update_batch, [field for field in provider_data.keys() if field != 'node_id'])
+        providers_to_update,
+        fields=[field for field in provider_data[0].keys() if field !=
+                'node_id']
+    )
 
 
 TESTNET_KEYS = [
@@ -76,81 +83,33 @@ def update_providers_info(node_props):
 from .utils import build_parser, print_env_info, format_usage  # noqa: E402
 
 
-def update_nodes_status(provider_id, is_online_now):
-    provider, created = Provider.objects.get_or_create(node_id=provider_id)
-
-    # Get the latest status from Redis
-    latest_status = r.get(f"provider:{provider_id}:status")
-
-    if latest_status is None:
-        # Status not found in Redis, fetch the latest status from the database
-        last_status = NodeStatusHistory.objects.filter(
-            provider=provider).last()
-        if not last_status or last_status.is_online != is_online_now:
-            # Create a new status entry if there's a change in status
-            NodeStatusHistory.objects.create(
-                provider=provider, is_online=is_online_now)
-            provider.online = is_online_now
-            provider.save()
-    else:
-        # Compare the latest status from Redis with the current status
-        if latest_status.decode() != str(is_online_now):
-            # Status has changed, update the database and Node.online field
-            NodeStatusHistory.objects.create(
-                provider=provider, is_online=is_online_now)
-            provider.online = is_online_now
-            provider.save()
-
-    # Store the current status in Redis for future lookups
-    r.set(f"provider:{provider_id}:status", str(is_online_now))
-
-
-@app.task(queue='uptime', options={'queue': 'uptime', 'routing_key': 'uptime'})
-def update_nodes_data(node_props):
-    r = redis.Redis(host='redis', port=6379, db=0)
-
-    for props in node_props:
-        issuer_id = props['node_id']
-        is_online_now = check_node_status(issuer_id)
-        print(f"Updating NodeStatus for {issuer_id} with is_online_now={is_online_now}")
-        try:
-            update_nodes_status(issuer_id, is_online_now)
-        except Exception as e:
-            print(f"Error updating NodeStatus for {issuer_id}: {e}")
-
-    provider_ids_in_props = {props['node_id'] for props in node_props}
-    previously_online_providers_ids = Provider.objects.filter(
-        nodestatushistory__is_online=True
-    ).distinct().values_list('node_id', flat=True)
-
-    provider_ids_not_in_scan = set(
-        previously_online_providers_ids) - provider_ids_in_props
-
-    for issuer_id in provider_ids_not_in_scan:
-        is_online_now = check_node_status(issuer_id)
-        print(f"Verifying NodeStatus for {issuer_id} with is_online_now={is_online_now}")
-        try:
-            update_nodes_status(issuer_id, is_online_now)
-        except Exception as e:
-            print(f"Error verifying/updating NodeStatus for {issuer_id}: {e}")
-
-
 def check_node_status(issuer_id):
+    node_id_no_prefix = issuer_id[2:] if issuer_id.startswith(
+        '0x') else issuer_id
+    url = f"http://yacn2.dev.golem.network:9000/nodes/{node_id_no_prefix}"
     try:
-        process = subprocess.run(
-            ["yagna", "net", "find", issuer_id],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            timeout=5  # 5-second timeout for the subprocess
-        )
-
-        # Process finished, return True if it was successful and "seen:" is in the output
-        return process.returncode == 0 and "seen:" in process.stdout.decode()
-    except subprocess.TimeoutExpired as e:
-        print("Timeout reached while checking node status", e)
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+        data = response.json()
+        node_key = issuer_id.lower()
+        node_info = data.get(node_key)
+
+        if node_info:
+            if isinstance(node_info, list):
+                if node_info == [] or node_info == [None]:
+                    return False
+                else:
+                    return any('seen' in item for item in node_info if item)
+            else:
+                return False
+        else:
+            return False
+    except requests.exceptions.RequestException as e:
+        print(
+            f"HTTP request exception when checking node status for {issuer_id}: {e}")
         return False
     except Exception as e:
-        print(f"Unexpected error checking node status: {e}")
+        print(f"Unexpected error checking node status for {issuer_id}: {e}")
         return False
 
 
@@ -200,5 +159,4 @@ async def monitor_nodes_status(subnet_tag: str = "public"):
         print("Scan timeout reached")
 
     # Delay update_nodes_data call using Celery
-    update_nodes_data.delay(node_props)
     update_providers_info.delay(node_props)
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/scoring.py b/docker-stack/golem-reputation-backend/reputation-backend/api/scoring.py
index 8456460..fe1fe56 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/api/scoring.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api/scoring.py
@@ -25,7 +25,7 @@ def penalty_weight(deviation):
 def calculate_uptime(node_id, node=None):
     if node is None:
         node = Provider.objects.get(node_id=node_id)
-    statuses = NodeStatusHistory.objects.filter(provider=node).order_by("timestamp")
+    statuses = NodeStatusHistory.objects.filter(node_id=node_id).order_by("timestamp")
 
     online_duration = timedelta(0)
     last_online_time = None
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/tasks.py b/docker-stack/golem-reputation-backend/reputation-backend/api/tasks.py
index ea73983..f9233c6 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/api/tasks.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api/tasks.py
@@ -70,9 +70,8 @@ def update_provider_scores(network):
     r = redis.Redis(host='redis', port=6379, db=0)
     ten_days_ago = timezone.now() - timedelta(days=10)
     recent_online_providers = NodeStatusHistory.objects.filter(
-        is_online=True).order_by('provider', '-timestamp').distinct('provider')
-    online_provider_ids = [
-        status.provider_id for status in recent_online_providers]
+        is_online=True).order_by('node_id', '-timestamp').distinct('node_id')
+    online_provider_ids = [status.node_id for status in recent_online_providers]
     providers = Provider.objects.filter(node_id__in=online_provider_ids, network=network).annotate(
         success_count=Count('taskcompletion', filter=Q(
             taskcompletion__is_successful=True, taskcompletion__timestamp__gte=ten_days_ago)),
@@ -241,7 +240,7 @@ def get_blacklisted_operators():
         latest=Max('timestamp', filter=Q(is_online=True))
     ).filter(
         timestamp=F('latest'), is_online=True
-    ).values_list('provider_id', flat=True)
+    ).values_list('node_id', flat=True)
 
     providers = Provider.objects.filter(
         node_id__in=latest_online_statuses
@@ -406,4 +405,73 @@ def delete_old_ping_results():
 
     # Optionally, you can log the number of deleted records or return it
     print(
-        f"Deleted {count_ping_results} PingResult records older than 30 days.")
\ No newline at end of file
+        f"Deleted {count_ping_results} PingResult records older than 30 days.")
+    
+import requests 
+from .utils import check_node_status
+from django.db import transaction
+r = redis.Redis(host='redis', port=6379, db=0)
+
+@app.task
+def fetch_and_update_relay_nodes_online_status():
+    base_url = "http://yacn2.dev.golem.network:9000/nodes/"
+    current_online_nodes = set()
+    nodes_to_update = []
+
+    for prefix in range(256):
+        try:
+            response = requests.get(f"{base_url}{prefix:02x}", timeout=5)
+            response.raise_for_status()
+            data = response.json()
+
+            for node_id, sessions in data.items():
+                node_id = node_id.strip().lower()
+                is_online = bool(sessions) and any('seen' in item for item in sessions if item)
+                current_online_nodes.add(node_id)
+                nodes_to_update.append((node_id, is_online))
+
+        except requests.RequestException as e:
+            print(f"Error fetching data for prefix {prefix:02x}: {e}")
+
+    # Bulk update node statuses
+    bulk_update_node_statuses.delay(nodes_to_update)
+
+    # Check providers that were previously online but not found in the current scan
+    previously_online = set(NodeStatusHistory.objects.filter(
+        is_online=True
+    ).order_by('node_id', '-timestamp').distinct('node_id').values_list('node_id', flat=True))
+
+    missing_nodes = previously_online - current_online_nodes
+    if missing_nodes:
+        check_missing_nodes.delay(list(missing_nodes))
+
+@app.task
+def bulk_update_node_statuses(nodes_data):
+    status_history_to_create = []
+    redis_updates = {}
+
+    for node_id, is_online in nodes_data:
+        latest_status = r.get(f"provider:{node_id}:status")
+        
+        if latest_status is None or latest_status.decode() != str(is_online):
+            status_history_to_create.append(
+                NodeStatusHistory(node_id=node_id, is_online=is_online)
+            )
+            redis_updates[f"provider:{node_id}:status"] = str(is_online)
+            
+
+    if status_history_to_create:
+        with transaction.atomic():
+            NodeStatusHistory.objects.bulk_create(status_history_to_create)
+
+    if redis_updates:
+        r.mset(redis_updates)
+
+@app.task
+def check_missing_nodes(missing_nodes):
+    nodes_to_update = []
+    for node_id in missing_nodes:
+        is_online = check_node_status(node_id)
+        nodes_to_update.append((node_id, is_online))
+    
+    bulk_update_node_statuses(nodes_to_update)
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api/utils.py b/docker-stack/golem-reputation-backend/reputation-backend/api/utils.py
index 9246928..47033d9 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/api/utils.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api/utils.py
@@ -105,4 +105,42 @@ def run_golem_example(example_main, log_file=None):
                 f"{TEXT_COLOR_YELLOW}Shutdown completed, thank you for waiting!{TEXT_COLOR_DEFAULT}"
             )
         except (asyncio.CancelledError, KeyboardInterrupt):
-            pass
\ No newline at end of file
+            pass
+
+
+import requests
+def check_node_status(node_id):
+    node_id_no_prefix = node_id[2:] if node_id.startswith('0x') else node_id
+    url = f"http://yacn2.dev.golem.network:9000/nodes/{node_id_no_prefix}"
+    try:
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+        data = response.json()
+        
+        # If the response is an empty dictionary, the node is considered offline
+        if not data:
+            return False
+        
+        node_key = node_id.lower()
+        node_info = data.get(node_key)
+
+        if node_info:
+            if isinstance(node_info, list):
+                if node_info == [] or node_info == [None]:
+                    return False
+                else:
+                    return any('seen' in item for item in node_info if item)
+            else:
+                return False
+        else:
+            return False
+    except requests.exceptions.RequestException as e:
+        print(url, node_id)
+        # Log the error and return False
+        print(f"HTTP request exception when checking node status for {node_id}: {e}")
+        return False
+    except Exception as e:
+        # Log the error and return False
+        print(url, node_id)
+        print(f"Unexpected error checking node status for {node_id}: {e}")
+        return False
\ No newline at end of file
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/api2/api.py b/docker-stack/golem-reputation-backend/reputation-backend/api2/api.py
index 0f6bd9a..371dd42 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/api2/api.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/api2/api.py
@@ -176,7 +176,7 @@ def filter_providers(
     ).annotate(
         latest_status=Subquery(
             NodeStatusHistory.objects.filter(
-                provider=OuterRef('pk')
+                node_id=OuterRef('node_id')
             ).order_by('-timestamp').values('is_online')[:1]
         )
     ).filter(latest_status=True)
@@ -868,7 +868,7 @@ def list_all_provider_scores(request):
 def get_score_overview(request):
     # Filter providers whose latest NodeStatusHistory is_online=True
     latest_status_subquery = NodeStatusHistory.objects.filter(
-        provider=OuterRef('pk')
+        node_id=OuterRef('node_id')
     ).order_by('-timestamp').values('is_online')[:1]
 
     providers = Provider.objects.annotate(
@@ -878,7 +878,7 @@ def get_score_overview(request):
     # Calculate uptime for each provider
     def calculate_uptime(provider):
         statuses = NodeStatusHistory.objects.filter(
-            provider=provider).order_by('timestamp')
+            node_id=provider.node_id).order_by('timestamp')
 
         online_duration = timedelta(0)
         last_online_time = None
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/core/celery.py b/docker-stack/golem-reputation-backend/reputation-backend/core/celery.py
index 31f59ee..bf01c8e 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/core/celery.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/core/celery.py
@@ -15,8 +15,14 @@
 
 @app.on_after_finalize.connect
 def setup_periodic_tasks(sender, **kwargs):
-    from api.tasks import monitor_nodes_task, ping_providers_task, process_offers_from_redis, update_provider_scores, get_blacklisted_operators, get_blacklisted_providers, delete_old_ping_results
+    from api.tasks import monitor_nodes_task, ping_providers_task, process_offers_from_redis, update_provider_scores, get_blacklisted_operators, get_blacklisted_providers, delete_old_ping_results, fetch_and_update_relay_nodes_online_status
     from stats.tasks import populate_daily_provider_stats, cache_provider_success_ratio, cache_provider_uptime, cache_cpu_performance_ranking, cache_gpu_performance_ranking
+    sender.add_periodic_task(
+        30.0,
+        fetch_and_update_relay_nodes_online_status.s(),
+        queue="default",
+        options={"queue": "default", "routing_key": "default"},
+    )
     sender.add_periodic_task(
         crontab(minute=0, hour=0),  # daily at midnight
         cache_provider_success_ratio.s(),
diff --git a/docker-stack/golem-reputation-backend/reputation-backend/stats/tasks.py b/docker-stack/golem-reputation-backend/reputation-backend/stats/tasks.py
index edfc331..9105839 100644
--- a/docker-stack/golem-reputation-backend/reputation-backend/stats/tasks.py
+++ b/docker-stack/golem-reputation-backend/reputation-backend/stats/tasks.py
@@ -70,18 +70,18 @@ def populate_daily_provider_stats():
 
 @app.task
 def cache_provider_uptime():
-
     # Get the latest online status for each provider
     latest_statuses = NodeStatusHistory.objects.filter(
         timestamp=Subquery(
-            NodeStatusHistory.objects.filter(provider=OuterRef('provider'))
+            NodeStatusHistory.objects.filter(node_id=OuterRef('node_id'))
             .order_by('-timestamp')
             .values('timestamp')[:1]
         )
     )
 
-    provider_ids = [status.provider_id for status in latest_statuses if status.is_online]
-
+    # Get online node_ids that also exist in the Provider model
+    online_node_ids = [status.node_id for status in latest_statuses if status.is_online]
+    existing_providers = Provider.objects.filter(node_id__in=online_node_ids).values_list('node_id', flat=True)
 
     uptime_data = {
         '100-90': 0,
@@ -92,20 +92,23 @@ def cache_provider_uptime():
         '20-0': 0
     }
     
-    for provider_id in provider_ids:
-        uptime_percentage = calculate_uptime(provider_id)
-        if uptime_percentage >= 90:
-            uptime_data['100-90'] += 1
-        elif uptime_percentage >= 80:
-            uptime_data['90-80'] += 1
-        elif uptime_percentage >= 60:
-            uptime_data['80-60'] += 1
-        elif uptime_percentage >= 40:
-            uptime_data['60-40'] += 1
-        elif uptime_percentage >= 20:
-            uptime_data['40-20'] += 1
-        else:
-            uptime_data['20-0'] += 1
+    for provider_id in existing_providers:
+        try:
+            uptime_percentage = calculate_uptime(provider_id)
+            if uptime_percentage >= 90:
+                uptime_data['100-90'] += 1
+            elif uptime_percentage >= 80:
+                uptime_data['90-80'] += 1
+            elif uptime_percentage >= 60:
+                uptime_data['80-60'] += 1
+            elif uptime_percentage >= 40:
+                uptime_data['60-40'] += 1
+            elif uptime_percentage >= 20:
+                uptime_data['40-20'] += 1
+            else:
+                uptime_data['20-0'] += 1
+        except Exception as e:
+            print(f"Error calculating uptime for provider {provider_id}: {str(e)}")
 
     redis_client.set('stats_provider_uptime', json.dumps(uptime_data))
 
@@ -115,14 +118,16 @@ def cache_provider_success_ratio():
     # Get the latest online status for each provider
     latest_statuses = NodeStatusHistory.objects.filter(
         timestamp=Subquery(
-            NodeStatusHistory.objects.filter(provider=OuterRef('provider'))
+            NodeStatusHistory.objects.filter(node_id=OuterRef('node_id'))
             .order_by('-timestamp')
             .values('timestamp')[:1]
         )
     )
 
-    provider_ids = [status.provider_id for status in latest_statuses if status.is_online]
+    online_node_ids = [status.node_id for status in latest_statuses if status.is_online]
 
+    # Get providers that exist in the Provider model and are online
+    existing_providers = Provider.objects.filter(node_id__in=online_node_ids)
 
     # Calculate success ratios
     success_ratio_data = {
@@ -134,15 +139,12 @@ def cache_provider_success_ratio():
         '20-0': 0
     }
 
-    for provider_id in provider_ids:
-        provider = Provider.objects.filter(node_id=provider_id).annotate(
-            success_count=Count('taskcompletion', filter=Q(
-                taskcompletion__is_successful=True)),
-            total_count=Count('taskcompletion'),
-        ).first()
-
-        if provider and provider.total_count > 0:
-            success_ratio = provider.success_count / provider.total_count * 100
+    for provider in existing_providers.annotate(
+        success_count=Count('taskcompletion', filter=Q(taskcompletion__is_successful=True)),
+        total_count=Count('taskcompletion')
+    ):
+        if provider.total_count > 0:
+            success_ratio = (provider.success_count / provider.total_count) * 100
             if success_ratio >= 90:
                 success_ratio_data['100-90'] += 1
             elif success_ratio >= 80:
@@ -156,8 +158,11 @@ def cache_provider_success_ratio():
             else:
                 success_ratio_data['20-0'] += 1
 
-    redis_client.set('stats_provider_success_ratio',
-                     json.dumps(success_ratio_data))
+    redis_client.set('stats_provider_success_ratio', json.dumps(success_ratio_data))
+    
+
+
+    
 
 
 from django.db.models import Max, F, Subquery, OuterRef, Prefetch