do not timeout on provisioning for cluster reassignment

internal/dinosaur/pkg/api/dbapi/central_request_types.go

@@ -97,6 +97,11 @@ type CentralRequest struct {
 	// After a grace period, the Central instance will be marked for deletion, its status will be set to 'deprovision'.
 	ExpiredAt sql.NullTime `json:"expired_at"`
 
+	// EnteredProvisioning is the timestamp when Central status was set to provisioning.
+	// It is required to track the provisioning duration and timeout on re-provision operations
+	// when the creation time is not reliable because it was way back in the past.
+	EnteredProvisionig sql.NullTime `json:"entered_provisioning"`
+
 	// Traits is a set of random strings assigned to an instance. Some traits
 	// can be hardcoded, and change some processing parameters.
 	Traits pq.StringArray `json:"traits" gorm:"type:text[]"`

internal/dinosaur/pkg/migrations/20241022160000_add_entered_provisioning_to_central_request.go

@@ -0,0 +1,36 @@
+package migrations
+
+// Migrations should NEVER use types from other packages. Types can change
+// and then migrations run on a _new_ database will fail or behave unexpectedly.
+// Instead of importing types, always re-create the type in the migration, as
+// is done here, even though the same type is defined in pkg/api
+
+import (
+	"database/sql"
+
+	"github.com/go-gormigrate/gormigrate/v2"
+	"github.com/pkg/errors"
+	"github.com/stackrox/acs-fleet-manager/pkg/db"
+	"gorm.io/gorm"
+)
+
+func addEnteredProvisioningToCentralRequest() *gormigrate.Migration {
+	type CentralRequest struct {
+		db.Model
+		EnteredProvisioning sql.NullTime `json:"entered_provisioning"`
+	}
+	migrationID := "20240703120000"
+
+	return &gormigrate.Migration{
+		ID: migrationID,
+		Migrate: func(tx *gorm.DB) error {
+			return addColumnIfNotExists(tx, &CentralRequest{}, "entered_provisioning")
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return errors.Wrap(
+				tx.Migrator().DropColumn(&CentralRequest{}, "entered_provisioning"),
+				"failed to drop entered_provisioning column",
+			)
+		},
+	}
+}

internal/dinosaur/pkg/migrations/migrations.go

@@ -55,6 +55,7 @@ func getMigrations() []*gormigrate.Migration {
 		addAlternateUserIDFieldToCentralRequests(),
 		addTraitsFieldToCentralRequests(),
 		addSecretDataSha256SumToCentralRequest(),
+		addEnteredProvisioningToCentralRequest(),
 	}
 }
 

internal/dinosaur/pkg/services/dinosaur.go

@@ -387,12 +387,14 @@ func (k *dinosaurService) PrepareDinosaurRequest(dinosaurRequest *dbapi.CentralR
 	}
 
 	// UpdateIgnoreNils the fields of the CentralRequest record in the database.
+	now := time.Now()
 	updatedCentralRequest := &dbapi.CentralRequest{
 		Meta: api.Meta{
 			ID: dinosaurRequest.ID,
 		},
-		OrganisationName: orgName,
-		Status:           dinosaurConstants.CentralRequestStatusProvisioning.String(),
+		OrganisationName:   orgName,
+		Status:             dinosaurConstants.CentralRequestStatusProvisioning.String(),
+		EnteredProvisionig: dbapi.TimePtrToNullTime(&now),
 	}
 	if err := k.UpdateIgnoreNils(updatedCentralRequest); err != nil {
 		return errors.NewWithCause(errors.ErrorGeneral, err, "failed to update central request")
@@ -901,13 +903,16 @@ func (k *dinosaurService) AssignCluster(ctx context.Context, centralID string, c
 	central.Routes = nil
 	central.RoutesCreationID = ""
 	central.Status = dinosaurConstants.CentralRequestStatusProvisioning.String()
+	now := time.Now()
+	central.EnteredProvisionig = dbapi.TimePtrToNullTime(&now)
 
 	return k.Updates(central, map[string]interface{}{
-		"cluster_id":         central.ClusterID,
-		"routes_created":     central.RoutesCreated,
-		"routes":             central.Routes,
-		"status":             central.Status,
-		"routes_creation_id": central.RoutesCreationID,
+		"cluster_id":           central.ClusterID,
+		"routes_created":       central.RoutesCreated,
+		"routes":               central.Routes,
+		"status":               central.Status,
+		"routes_creation_id":   central.RoutesCreationID,
+		"entered_provisioning": central.EnteredProvisionig,
 	})
 }
 

internal/dinosaur/pkg/workers/dinosaurmgrs/timeout.go

@@ -13,7 +13,12 @@ import (
 // FailIfTimeoutExceeded checks timeout on a central instance and moves it to failed if timeout is exceeded.
 // Returns true if timeout is exceeded, otherwise false.
 func FailIfTimeoutExceeded(centralService services.DinosaurService, timeout time.Duration, centralRequest *dbapi.CentralRequest) error {
-	if centralRequest.CreatedAt.Before(time.Now().Add(-timeout)) {
+	referencePoint := centralRequest.CreatedAt
+	if centralRequest.EnteredProvisionig.Valid {
+		referencePoint = centralRequest.EnteredProvisionig.Time
+	}
+
+	if referencePoint.Before(time.Now().Add(-timeout)) {
 		centralRequest.Status = constants2.CentralRequestStatusFailed.String()
 		centralRequest.FailedReason = "Creation time went over the timeout. Interrupting central initialization."