Merge branch 'staging' into update-fix/Clean_up

Updates from airqo staging

k8s/analytics/values-prod.yaml

@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-analytics-devices-summary-job
-  tag: prod-48e4e72c-1734023346
+  tag: prod-85d64e95-1734081670
 api:
   name: airqo-analytics-api
   label: analytics-api

k8s/analytics/values-stage.yaml

@@ -8,7 +8,7 @@ images:
     celeryWorker: eu.gcr.io/airqo-250220/airqo-stage-analytics-celery-worker
     reportJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-report-job
     devicesSummaryJob: eu.gcr.io/airqo-250220/airqo-stage-analytics-devices-summary-job
-  tag: stage-dd764c29-1733849460
+  tag: stage-1396ff4a-1734025613
 api:
   name: airqo-stage-analytics-api
   label: sta-alytics-api

k8s/auth-service/values-prod.yaml

@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-auth-api
-  tag: prod-ee15b958-1733833086
+  tag: prod-85d64e95-1734081670
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

k8s/auth-service/values-stage.yaml

@@ -6,7 +6,7 @@ app:
 replicaCount: 2
 image:
   repository: eu.gcr.io/airqo-250220/airqo-stage-auth-api
-  tag: stage-95dea6dd-1733435752
+  tag: stage-c701f88b-1734080836
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

k8s/device-registry/values-prod.yaml

@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-device-registry-api
-  tag: prod-48e4e72c-1734023346
+  tag: prod-85d64e95-1734081670
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

k8s/exceedance/values-prod-airqo.yaml

@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/airqo-exceedance-job
-  tag: prod-48e4e72c-1734023346
+  tag: prod-85d64e95-1734081670
 nameOverride: ''
 fullnameOverride: ''

k8s/exceedance/values-prod-kcca.yaml

@@ -4,6 +4,6 @@ app:
   configmap: env-exceedance-production
 image:
   repository: eu.gcr.io/airqo-250220/kcca-exceedance-job
-  tag: prod-48e4e72c-1734023346
+  tag: prod-85d64e95-1734081670
 nameOverride: ''
 fullnameOverride: ''

k8s/predict/values-prod.yaml

@@ -7,7 +7,7 @@ images:
     predictJob: eu.gcr.io/airqo-250220/airqo-predict-job
     trainJob: eu.gcr.io/airqo-250220/airqo-train-job
     predictPlaces: eu.gcr.io/airqo-250220/airqo-predict-places-air-quality
-  tag: prod-48e4e72c-1734023346
+  tag: prod-85d64e95-1734081670
 api:
   name: airqo-prediction-api
   label: prediction-api

k8s/spatial/values-prod.yaml

@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-spatial-api
-  tag: prod-48e4e72c-1734023346
+  tag: prod-85d64e95-1734081670
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

k8s/website/values-prod.yaml

@@ -6,7 +6,7 @@ app:
 replicaCount: 3
 image:
   repository: eu.gcr.io/airqo-250220/airqo-website-api
-  tag: prod-48e4e72c-1734023346
+  tag: prod-85d64e95-1734081670
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

k8s/workflows/values-prod.yaml

@@ -10,7 +10,7 @@ images:
     initContainer: eu.gcr.io/airqo-250220/airqo-workflows-xcom
     redisContainer: eu.gcr.io/airqo-250220/airqo-redis
     containers: eu.gcr.io/airqo-250220/airqo-workflows
-  tag: prod-48e4e72c-1734023346
+  tag: prod-85d64e95-1734081670
 nameOverride: ''
 fullnameOverride: ''
 podAnnotations: {}

src/auth-service/models/Preference.js

@@ -182,7 +182,6 @@ const PreferenceSchema = new mongoose.Schema(
       type: ObjectId,
       required: [true, "user_id is required"],
       ref: "user",
-      unique: true,
     },
     site_ids: [
       {
@@ -213,48 +212,233 @@ PreferenceSchema.plugin(uniqueValidator, {
   message: `{VALUE} should be unique!`,
 });
 
-PreferenceSchema.pre("save", function (next) {
-  const fieldsToUpdate = [
-    "selected_sites",
-    "selected_grids",
-    "selected_cohorts",
-    "selected_devices",
-    "selected_airqlouds",
-  ];
-
-  const currentDate = new Date();
-
-  fieldsToUpdate.forEach((field) => {
-    if (this[field]) {
-      this[field] = Array.from(
-        new Set(
-          this[field].map((item) => ({
-            ...item,
-            createdAt: item.createdAt || currentDate,
-          }))
-        )
+PreferenceSchema.index({ user_id: 1, group_id: 1 }, { unique: true });
+
+PreferenceSchema.pre(
+  [
+    "save",
+    "create",
+    "update",
+    "findByIdAndUpdate",
+    "updateMany",
+    "updateOne",
+    "findOneAndUpdate",
+  ],
+  async function (next) {
+    try {
+      // Determine if this is a new document or an update
+      const isNew = this.isNew;
+      const updateData = this.getUpdate ? this.getUpdate() : this;
+
+      // Utility function to validate and process ObjectIds
+      const processObjectId = (id) => {
+        if (!id) return null;
+        if (id instanceof mongoose.Types.ObjectId) return id;
+        try {
+          return mongoose.Types.ObjectId(id);
+        } catch (error) {
+          logger.error(`Invalid ObjectId: ${id}`);
+          throw new Error(`Invalid ObjectId: ${id}`);
+        }
+      };
+
+      // Comprehensive ID fields processing
+      const idFieldsToProcess = [
+        "airqloud_id",
+        "airqloud_ids",
+        "grid_id",
+        "grid_ids",
+        "cohort_id",
+        "cohort_ids",
+        "network_id",
+        "network_ids",
+        "group_id",
+        "group_ids",
+        "site_ids",
+        "device_ids",
+      ];
+
+      idFieldsToProcess.forEach((field) => {
+        if (updateData[field]) {
+          if (Array.isArray(updateData[field])) {
+            updateData[field] = updateData[field].map(processObjectId);
+          } else {
+            updateData[field] = processObjectId(updateData[field]);
+          }
+        }
+      });
+
+      // Validate user_id
+      if (!updateData.user_id) {
+        return next(new Error("user_id is required"));
+      }
+      updateData.user_id = processObjectId(updateData.user_id);
+
+      // Set default values if not provided
+      const defaultFields = [
+        { field: "pollutant", default: "pm2_5" },
+        { field: "frequency", default: "hourly" },
+        { field: "chartType", default: "line" },
+        { field: "chartTitle", default: "Chart Title" },
+        { field: "chartSubTitle", default: "Chart SubTitle" },
+      ];
+
+      defaultFields.forEach(({ field, default: defaultValue }) => {
+        if (isNew && !updateData[field]) {
+          updateData[field] = defaultValue;
+        }
+      });
+
+      // Handle date fields
+      if (isNew) {
+        const currentDate = new Date();
+        updateData.startDate =
+          updateData.startDate || addWeeksToProvideDateTime(currentDate, -2);
+        updateData.endDate = updateData.endDate || currentDate;
+      }
+
+      // Validate and process period schema
+      if (updateData.period) {
+        const validPeriodFields = ["value", "label", "unitValue", "unit"];
+        const periodUpdate = {};
+
+        validPeriodFields.forEach((field) => {
+          if (updateData.period[field] !== undefined) {
+            periodUpdate[field] = updateData.period[field];
+          }
+        });
+
+        // Additional period validation
+        if (
+          periodUpdate.unitValue !== undefined &&
+          typeof periodUpdate.unitValue !== "number"
+        ) {
+          periodUpdate.unitValue = Number(periodUpdate.unitValue);
+        }
+
+        updateData.period = periodUpdate;
+      }
+
+      // Process and validate selected arrays with their specific schemas
+      const selectedArrayProcessors = {
+        selected_sites: (site) => {
+          const processedSite = { ...site };
+
+          // Validate and process ObjectIds
+          if (site._id) processedSite._id = processObjectId(site._id);
+          if (site.grid_id)
+            processedSite.grid_id = processObjectId(site.grid_id);
+
+          // Validate numeric fields
+          const numericFields = [
+            "latitude",
+            "longitude",
+            "approximate_latitude",
+            "approximate_longitude",
+          ];
+          numericFields.forEach((field) => {
+            if (processedSite[field] !== undefined) {
+              processedSite[field] = Number(processedSite[field]);
+            }
+          });
+
+          // Ensure createdAt is a valid date
+          processedSite.createdAt = site.createdAt || new Date();
+
+          // Validate string fields
+          const stringFields = [
+            "country",
+            "district",
+            "sub_county",
+            "parish",
+            "county",
+            "generated_name",
+            "name",
+            "city",
+            "formatted_name",
+            "region",
+            "search_name",
+          ];
+          stringFields.forEach((field) => {
+            if (processedSite[field]) {
+              processedSite[field] = String(processedSite[field]).trim();
+            }
+          });
+
+          // Ensure boolean fields
+          processedSite.isFeatured = !!site.isFeatured;
+
+          return processedSite;
+        },
+        selected_grids: (grid) => ({
+          _id: processObjectId(grid._id),
+          name: String(grid.name).trim(),
+          createdAt: grid.createdAt || new Date(),
+        }),
+        selected_cohorts: (cohort) => ({
+          _id: processObjectId(cohort._id),
+          name: String(cohort.name).trim(),
+          createdAt: cohort.createdAt || new Date(),
+        }),
+        selected_devices: (device) => ({
+          _id: processObjectId(device._id),
+          name: String(device.name).trim(),
+          createdAt: device.createdAt || new Date(),
+        }),
+        selected_airqlouds: (airqloud) => ({
+          _id: processObjectId(airqloud._id),
+          name: String(airqloud.name).trim(),
+          createdAt: airqloud.createdAt || new Date(),
+        }),
+      };
+
+      // Process selected arrays
+      Object.keys(selectedArrayProcessors).forEach((field) => {
+        if (updateData[field]) {
+          updateData[field] = updateData[field].map(
+            selectedArrayProcessors[field]
+          );
+        }
+      });
+
+      // Prepare $addToSet for array fields to prevent duplicates
+      const arrayFieldsToAddToSet = [
+        ...idFieldsToProcess,
+        "selected_sites",
+        "selected_grids",
+        "selected_devices",
+        "selected_cohorts",
+        "selected_airqlouds",
+      ];
+
+      arrayFieldsToAddToSet.forEach((field) => {
+        if (updateData[field]) {
+          updateData.$addToSet = updateData.$addToSet || {};
+          updateData.$addToSet[field] = {
+            $each: updateData[field],
+          };
+          delete updateData[field];
+        }
+      });
+
+      // Optional: Add comprehensive logging
+      console.log(
+        `Preprocessing preference document: ${isNew ? "New" : "Update"}`,
+        {
+          user_id: updateData.user_id,
+          pollutant: updateData.pollutant,
+          startDate: updateData.startDate,
+          endDate: updateData.endDate,
+        }
       );
-    }
-  });
-
-  const fieldsToAddToSet = [
-    "airqloud_ids",
-    "device_ids",
-    "cohort_ids",
-    "grid_ids",
-    "site_ids",
-    "network_ids",
-    "group_ids",
-  ];
-
-  fieldsToAddToSet.forEach((field) => {
-    if (this[field]) {
-      this[field] = Array.from(new Set(this[field].map((id) => id.toString())));
-    }
-  });
 
-  return next();
-});
+      next();
+    } catch (error) {
+      console.error("Error in Preference pre-hook:", error);
+      return next(error);
+    }
+  }
+);
 
 PreferenceSchema.methods = {
   toJSON() {