[PLAT-14785] Add REST APIs for job scheduler (auto-master failover)

Summary: Add REST APIs to view the internally scheduled jobs + some fixes based on the discussion on demo. Test Plan: 1. UTs passed for controller. 2. For the removal of failure based on waitForServer, the test was done after changing the timeouts for master heartbeat delays and follower lags. ``` 2024-07-30T22:23:03.521Z [error] 2d2a4440-edbd-401d-983f-b732973c2761 AutoMasterFailover.java:291 [auto-master-failover-executor-1] com.yugabyte.yw.commissioner.AutoMasterFailover Failing master 10.9.81.185 in universe 8a510164a4e645319c299c10398b1ae5 as hearbeat delay exceeds threshold 47202a5e-fc61-4222-adec-999204861d29ms 2024-07-30T22:23:03.524Z [info] 2d2a4440-edbd-401d-983f-b732973c2761 AutoMasterFailover.java:186 [auto-master-failover-executor-1] com.yugabyte.yw.commissioner.AutoMasterFailover Failed masters for universe 47202a5e-fc61-4222-adec-999204861d29: [yb-admin-nsingh-test-universe1-n1] ``` Reviewers: amalyshev, cwang, sanketh, #yba-api-review, sneelakantan Reviewed By: amalyshev, #yba-api-review, sneelakantan Subscribers: sneelakantan, yugaware Differential Revision: https://phorge.dev.yugabyte.com/D36938
yugabyte · Aug 9, 2024 · 9e7181f · 9e7181f
1 parent efd4cb7
commit 9e7181f
Show file tree

Hide file tree

Showing 67 changed files with 1,914 additions and 202 deletions.
diff --git a/managed/src/main/java/api/v2/controllers/JobSchedulerApiControllerImp.java b/managed/src/main/java/api/v2/controllers/JobSchedulerApiControllerImp.java
@@ -0,0 +1,58 @@
+// Copyright (c) YugaByte, Inc.
+
+package api.v2.controllers;
+
+import api.v2.handlers.JobSchedulerHandler;
+import api.v2.models.JobInstancePagedQuerySpec;
+import api.v2.models.JobInstancePagedResp;
+import api.v2.models.JobSchedule;
+import api.v2.models.JobSchedulePagedQuerySpec;
+import api.v2.models.JobSchedulePagedResp;
+import api.v2.models.JobScheduleSnoozeSpec;
+import api.v2.models.JobScheduleUpdateSpec;
+import com.google.inject.Inject;
+import java.util.UUID;
+import play.mvc.Http;
+import play.mvc.Http.Request;
+
+public class JobSchedulerApiControllerImp extends JobSchedulerApiControllerImpInterface {
+  @Inject private JobSchedulerHandler jobSchedulerHandler;
+
+  @Override
+  public JobSchedulePagedResp pageListJobSchedules(
+      Http.Request request, UUID cUUID, JobSchedulePagedQuerySpec jobSchedulePagedQuerySpec)
+      throws Exception {
+    return jobSchedulerHandler.pagedListJobSchedules(cUUID, jobSchedulePagedQuerySpec);
+  }
+
+  @Override
+  public JobSchedule deleteJobSchedule(Request request, UUID cUUID, UUID jUUID) throws Exception {
+    return jobSchedulerHandler.deleteJobSchedule(cUUID, jUUID);
+  }
+
+  @Override
+  public JobSchedule getJobSchedule(Request request, UUID cUUID, UUID jUUID) throws Exception {
+    return jobSchedulerHandler.getJobSchedule(cUUID, jUUID);
+  }
+
+  @Override
+  public JobInstancePagedResp pageListJobInstances(
+      Request request, UUID cUUID, UUID jUUID, JobInstancePagedQuerySpec jobInstancePagedQuerySpec)
+      throws Exception {
+    return jobSchedulerHandler.pageListJobInstances(cUUID, jUUID, jobInstancePagedQuerySpec);
+  }
+
+  @Override
+  public JobSchedule snoozeJobSchedule(
+      Request request, UUID cUUID, UUID jUUID, JobScheduleSnoozeSpec jobScheduleSnoozeSpec)
+      throws Exception {
+    return jobSchedulerHandler.snoozeJobSchedule(cUUID, jUUID, jobScheduleSnoozeSpec);
+  }
+
+  @Override
+  public JobSchedule updateJobSchedule(
+      Request request, UUID cUUID, UUID jUUID, JobScheduleUpdateSpec jobScheduleUpdateSpec)
+      throws Exception {
+    return jobSchedulerHandler.updateJobSchedule(cUUID, jUUID, jobScheduleUpdateSpec);
+  }
+}
diff --git a/managed/src/main/java/api/v2/controllers/UniverseApiControllerImp.java b/managed/src/main/java/api/v2/controllers/UniverseApiControllerImp.java
@@ -1,4 +1,5 @@
 // Copyright (c) YugaByte, Inc.
+
 package api.v2.controllers;
 
 import api.v2.handlers.UniverseManagementHandler;

diff --git a/managed/src/main/java/api/v2/handlers/JobSchedulerHandler.java b/managed/src/main/java/api/v2/handlers/JobSchedulerHandler.java
@@ -0,0 +1,116 @@
+// Copyright (c) Yugabyte, Inc.
+
+package api.v2.handlers;
+
+import static com.yugabyte.yw.models.helpers.CommonUtils.performPagedQuery;
+
+import api.v2.mappers.JobSchedulerMapper;
+import api.v2.models.JobInstancePagedQuerySpec;
+import api.v2.models.JobInstancePagedResp;
+import api.v2.models.JobSchedule;
+import api.v2.models.JobSchedulePagedQuerySpec;
+import api.v2.models.JobSchedulePagedResp;
+import api.v2.models.JobScheduleSnoozeSpec;
+import api.v2.models.JobScheduleUpdateSpec;
+import api.v2.utils.ApiControllerUtils;
+import com.yugabyte.yw.forms.JobScheduleUpdateForm;
+import com.yugabyte.yw.models.JobInstance;
+import com.yugabyte.yw.models.filters.JobInstanceFilter;
+import com.yugabyte.yw.models.filters.JobScheduleFilter;
+import com.yugabyte.yw.models.helpers.schedule.ScheduleConfig;
+import com.yugabyte.yw.models.paging.JobInstancePagedQuery;
+import com.yugabyte.yw.models.paging.JobInstancePagedResponse;
+import com.yugabyte.yw.models.paging.JobSchedulePagedQuery;
+import com.yugabyte.yw.models.paging.JobSchedulePagedResponse;
+import com.yugabyte.yw.models.paging.PagedQuery.SortDirection;
+import com.yugabyte.yw.scheduler.JobScheduler;
+import io.ebean.Query;
+import java.util.Optional;
+import java.util.UUID;
+import javax.inject.Inject;
+
+public class JobSchedulerHandler extends ApiControllerUtils {
+
+  private JobScheduler jobScheduler;
+
+  @Inject
+  public JobSchedulerHandler(JobScheduler jobScheduler) {
+    this.jobScheduler = jobScheduler;
+  }
+
+  public JobSchedulePagedResp pagedListJobSchedules(
+      UUID customerUuid, JobSchedulePagedQuerySpec pagedQuerySpec) {
+    JobSchedulePagedQuery pagedQuery =
+        JobSchedulerMapper.INSTANCE.toJobSchedulePagedQuery(pagedQuerySpec);
+    if (pagedQuery.getSortBy() == null) {
+      pagedQuery.setSortBy(com.yugabyte.yw.models.JobSchedule.SortBy.name);
+      pagedQuery.setDirection(SortDirection.DESC);
+    }
+    if (pagedQuery.getFilter() == null) {
+      pagedQuery.setFilter(JobScheduleFilter.builder().build());
+    }
+    Query<com.yugabyte.yw.models.JobSchedule> query =
+        com.yugabyte.yw.models.JobSchedule.createQuery(customerUuid, pagedQuery.getFilter())
+            .query();
+    JobSchedulePagedResponse response =
+        performPagedQuery(query, pagedQuery, JobSchedulePagedResponse.class);
+    return JobSchedulerMapper.INSTANCE.toJobSchedulePagedResp(response);
+  }
+
+  public JobSchedule getJobSchedule(UUID customerUuid, UUID jobScheduleUuid) {
+    return JobSchedulerMapper.INSTANCE.toJobSchedule(
+        com.yugabyte.yw.models.JobSchedule.getOrBadRequest(customerUuid, jobScheduleUuid));
+  }
+
+  public JobSchedule updateJobSchedule(
+      UUID cUUID, UUID jUUID, JobScheduleUpdateSpec jobScheduleUpdateSpec) throws Exception {
+    JobScheduleUpdateForm updateForm =
+        JobSchedulerMapper.INSTANCE.toJobScheduleUpdateForm(jobScheduleUpdateSpec);
+    ScheduleConfig.ScheduleConfigBuilder builder =
+        com.yugabyte.yw.models.JobSchedule.getOrBadRequest(cUUID, jUUID)
+            .getScheduleConfig()
+            .toBuilder();
+    builder.intervalSecs(updateForm.intervalSecs);
+    builder.disabled(updateForm.disabled);
+    builder.type(updateForm.type);
+    return JobSchedulerMapper.INSTANCE.toJobSchedule(
+        jobScheduler.updateSchedule(jUUID, builder.build()));
+  }
+
+  public JobSchedule snoozeJobSchedule(
+      UUID cUUID, UUID jUUID, JobScheduleSnoozeSpec jobScheduleSnoozeSpec) throws Exception {
+    com.yugabyte.yw.models.JobSchedule jobSchedule =
+        com.yugabyte.yw.models.JobSchedule.getOrBadRequest(cUUID, jUUID);
+    return JobSchedulerMapper.INSTANCE.toJobSchedule(
+        jobScheduler.snooze(jobSchedule.getUuid(), jobScheduleSnoozeSpec.getSnoozeSecs()));
+  }
+
+  public JobSchedule deleteJobSchedule(UUID cUUID, UUID jUUID) throws Exception {
+    Optional<com.yugabyte.yw.models.JobSchedule> optional =
+        com.yugabyte.yw.models.JobSchedule.maybeGet(cUUID, jUUID);
+    if (optional.isPresent()) {
+      jobScheduler.deleteSchedule(optional.get().getUuid());
+      return JobSchedulerMapper.INSTANCE.toJobSchedule(optional.get());
+    }
+    return new JobSchedule();
+  }
+
+  public JobInstancePagedResp pageListJobInstances(
+      UUID cUUID, UUID jUUID, JobInstancePagedQuerySpec jobInstancePagedQuerySpec)
+      throws Exception {
+    com.yugabyte.yw.models.JobSchedule.getOrBadRequest(cUUID, jUUID);
+    JobInstancePagedQuery pagedQuery =
+        JobSchedulerMapper.INSTANCE.toJobInstancePagedQuery(jobInstancePagedQuerySpec);
+    if (pagedQuery.getSortBy() == null) {
+      pagedQuery.setSortBy(JobInstance.SortBy.jobScheduleUuid);
+      pagedQuery.setDirection(SortDirection.DESC);
+    }
+    if (pagedQuery.getFilter() == null) {
+      pagedQuery.setFilter(JobInstanceFilter.builder().build());
+    }
+    Query<JobInstance> query = JobInstance.createQuery(jUUID, pagedQuery.getFilter()).query();
+    JobInstancePagedResponse response =
+        performPagedQuery(query, pagedQuery, JobInstancePagedResponse.class);
+    return JobSchedulerMapper.INSTANCE.toJobInstancePagedResp(response);
+  }
+}
diff --git a/managed/src/main/java/api/v2/handlers/UniverseManagementHandler.java b/managed/src/main/java/api/v2/handlers/UniverseManagementHandler.java
@@ -1,4 +1,5 @@
 // Copyright (c) Yugabyte, Inc.
+
 package api.v2.handlers;
 
 import api.v2.mappers.ClusterMapper;

diff --git a/managed/src/main/java/api/v2/mappers/ClusterMapper.java b/managed/src/main/java/api/v2/mappers/ClusterMapper.java
@@ -1,4 +1,5 @@
 // Copyright (c) YugaByte, Inc.
+
 package api.v2.mappers;
 
 import api.v2.models.ClusterAddSpec;

diff --git a/managed/src/main/java/api/v2/mappers/CommunicationPortsMapper.java b/managed/src/main/java/api/v2/mappers/CommunicationPortsMapper.java
@@ -1,3 +1,5 @@
+// Copyright (c) YugaByte, Inc.
+
 package api.v2.mappers;
 
 import api.v2.models.CommunicationPortsSpec;

diff --git a/managed/src/main/java/api/v2/mappers/JobSchedulerMapper.java b/managed/src/main/java/api/v2/mappers/JobSchedulerMapper.java
@@ -0,0 +1,111 @@
+// Copyright (c) YugaByte, Inc.
+
+package api.v2.mappers;
+
+import api.v2.models.JobConfigSpec;
+import api.v2.models.JobInstancePagedQuerySpec;
+import api.v2.models.JobInstancePagedResp;
+import api.v2.models.JobSchedule;
+import api.v2.models.JobScheduleConfigSpec;
+import api.v2.models.JobScheduleInfo;
+import api.v2.models.JobSchedulePagedQuerySpec;
+import api.v2.models.JobSchedulePagedResp;
+import api.v2.models.JobScheduleSpec;
+import api.v2.models.JobScheduleType;
+import api.v2.models.JobScheduleUpdateSpec;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.yugabyte.yw.forms.JobScheduleUpdateForm;
+import com.yugabyte.yw.models.helpers.schedule.ScheduleConfig.ScheduleType;
+import com.yugabyte.yw.models.paging.JobInstancePagedQuery;
+import com.yugabyte.yw.models.paging.JobInstancePagedResponse;
+import com.yugabyte.yw.models.paging.JobSchedulePagedQuery;
+import com.yugabyte.yw.models.paging.JobSchedulePagedResponse;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.Date;
+import java.util.Map;
+import org.mapstruct.EnumMapping;
+import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
+import org.mapstruct.MappingConstants;
+import org.mapstruct.factory.Mappers;
+import play.libs.Json;
+
+@Mapper(config = CentralConfig.class)
+public interface JobSchedulerMapper {
+  final JobSchedulerMapper INSTANCE = Mappers.getMapper(JobSchedulerMapper.class);
+
+  @Mapping(target = "needTotalCount", constant = "true")
+  JobSchedulePagedQuery toJobSchedulePagedQuery(JobSchedulePagedQuerySpec pagedQuerySpec);
+
+  @EnumMapping(
+      nameTransformationStrategy = MappingConstants.PREFIX_TRANSFORMATION,
+      configuration = "FIXED_")
+  ScheduleType toScheduleType(JobScheduleType type);
+
+  @EnumMapping(
+      nameTransformationStrategy = MappingConstants.STRIP_PREFIX_TRANSFORMATION,
+      configuration = "FIXED_")
+  JobScheduleType toJobScheduleType(ScheduleType type);
+
+  default com.yugabyte.yw.models.JobSchedule.SortBy toJobScheduleSortBy(
+      JobSchedulePagedQuerySpec.SortByEnum sortByEnum) {
+    return sortByEnum == null
+        ? null
+        : com.yugabyte.yw.models.JobSchedule.SortBy.valueOf(sortByEnum.toString());
+  }
+
+  default JobSchedulePagedQuerySpec.SortByEnum toJobScheduleSortByEnum(
+      com.yugabyte.yw.models.JobSchedule.SortBy sortBy) {
+    return JobSchedulePagedQuerySpec.SortByEnum.fromValue(sortBy.name());
+  }
+
+  default com.yugabyte.yw.models.JobInstance.SortBy toJobInstanceSortBy(
+      JobInstancePagedQuerySpec.SortByEnum sortByEnum) {
+    return sortByEnum == null
+        ? null
+        : com.yugabyte.yw.models.JobInstance.SortBy.valueOf(sortByEnum.toString());
+  }
+
+  default JobInstancePagedQuerySpec.SortByEnum toJobInstanceSortByEnum(
+      com.yugabyte.yw.models.JobInstance.SortBy sortBy) {
+    return sortBy == null ? null : JobInstancePagedQuerySpec.SortByEnum.fromValue(sortBy.name());
+  }
+
+  default JobConfigSpec toJobConfigSpec(com.yugabyte.yw.models.helpers.schedule.JobConfig config) {
+    JobConfigSpec jobConfigSpec = new JobConfigSpec();
+    jobConfigSpec.setClassname(config.getClass().getName());
+    jobConfigSpec.setConfig(
+        Json.mapper()
+            .convertValue(Json.toJson(config), new TypeReference<Map<String, Object>>() {}));
+    return jobConfigSpec;
+  }
+
+  default OffsetDateTime toOffsetDateTime(Date date) {
+    return date == null ? null : date.toInstant().atOffset(ZoneOffset.UTC);
+  }
+
+  default JobSchedule toJobSchedule(com.yugabyte.yw.models.JobSchedule jobSchedule) {
+    JobSchedule v2JobSchedule = new JobSchedule();
+    JobScheduleSpec spec = new JobScheduleSpec();
+    spec.setJobConfig(toJobConfigSpec(jobSchedule.getJobConfig()));
+    spec.setScheduleConfig(toJobScheduleSpec(jobSchedule.getScheduleConfig()));
+    v2JobSchedule.setSpec(spec);
+    v2JobSchedule.setInfo(toJobScheduleInfo(jobSchedule));
+    return v2JobSchedule;
+  }
+
+  JobScheduleInfo toJobScheduleInfo(com.yugabyte.yw.models.JobSchedule jobSchedule);
+
+  JobScheduleConfigSpec toJobScheduleSpec(
+      com.yugabyte.yw.models.helpers.schedule.ScheduleConfig scheduleConfig);
+
+  JobSchedulePagedResp toJobSchedulePagedResp(JobSchedulePagedResponse response);
+
+  JobScheduleUpdateForm toJobScheduleUpdateForm(JobScheduleUpdateSpec jobScheduleUpdateSpec);
+
+  @Mapping(target = "needTotalCount", constant = "true")
+  JobInstancePagedQuery toJobInstancePagedQuery(JobInstancePagedQuerySpec pagedQuerySpec);
+
+  JobInstancePagedResp toJobInstancePagedResp(JobInstancePagedResponse response);
+}
diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/AutoMasterFailover.java b/managed/src/main/java/com/yugabyte/yw/commissioner/AutoMasterFailover.java
@@ -310,11 +310,13 @@ Set<String> getFailedMastersForUniverse(Universe universe, YBClient ybClient) {
                   failedMasters.add(node.getNodeName());
                 }
               } else {
-                log.error(
-                    "Failing master {} in universe {} as it is not alive",
-                    ipAddress,
-                    universe.getUniverseUUID());
-                failedMasters.add(node.getNodeName());
+                // Cannot decide at this time, wait for heartbeat delay to catch it.
+                String errMsg =
+                    String.format(
+                        "Follower lag for master %s in universe %s cannot be fetched",
+                        ipAddress, universe.getUniverseUUID());
+                log.error(errMsg);
+                throw new RuntimeException(errMsg);
               }
             });
     return failedMasters;
@@ -350,14 +352,19 @@ public Action getAllowedMasterFailoverAction(Customer customer, Universe univers
       boolean autoSyncMasterAddrs =
           universe.getNodes().stream().anyMatch(n -> n.autoSyncMasterAddrs);
       if (autoSyncMasterAddrs) {
+        log.info("Sync master addresses is pending for universe {}", universe.getUniverseUUID());
         // Always sync even if another master may have failed.
         // TODO we may want to run this earlier if at least one is up.
-        return areAllTabletServersAlive(universe)
-            ? Action.builder()
-                .actionType(ActionType.SUBMIT)
-                .taskType(TaskType.SyncMasterAddresses)
-                .build()
-            : Action.builder().actionType(ActionType.NONE).build();
+        if (areAllTabletServersAlive(universe)) {
+          return Action.builder()
+              .actionType(ActionType.SUBMIT)
+              .taskType(TaskType.SyncMasterAddresses)
+              .build();
+        }
+        log.warn(
+            "Sync master addresses is skipped as some tservers not alive for universe {}",
+            universe.getUniverseUUID());
+        return Action.builder().actionType(ActionType.NONE).build();
       }
       String failedNodeName = validateAndGetFailedNodeName(customer, universe);
       return failedNodeName == null