Add document hard deletion functionality to housekeeping tasks.

cmd/yorkie/server.go

@@ -38,9 +38,11 @@ var (
 	flagConfPath string
 	flagLogLevel string
 
-	adminTokenDuration        time.Duration
-	housekeepingInterval      time.Duration
-	clientDeactivateThreshold string
+	adminTokenDuration                       time.Duration
+	housekeepingIntervalDeactivateCandidates time.Duration
+	housekeepingIntervalDeleteDocuments      time.Duration
+	documentHardDeletionGracefulPeriod       time.Duration
+	clientDeactivateThreshold                string
 
 	mongoConnectionURI     string
 	mongoConnectionTimeout time.Duration
@@ -69,7 +71,9 @@ func newServerCmd() *cobra.Command {
 			conf.Backend.AuthWebhookCacheUnauthTTL = authWebhookCacheUnauthTTL.String()
 			conf.Backend.ProjectInfoCacheTTL = projectInfoCacheTTL.String()
 
-			conf.Housekeeping.Interval = housekeepingInterval.String()
+			conf.Housekeeping.IntervalDeactivateCandidates = housekeepingIntervalDeactivateCandidates.String()
+			conf.Housekeeping.IntervalDeleteDocuments = housekeepingIntervalDeleteDocuments.String()
+			conf.Housekeeping.DocumentHardDeletionGracefulPeriod = documentHardDeletionGracefulPeriod.String()
 
 			if mongoConnectionURI != "" {
 				conf.Mongo = &mongo.Config{
@@ -194,17 +198,35 @@ func init() {
 		"Enable runtime profiling data via HTTP server.",
 	)
 	cmd.Flags().DurationVar(
-		&housekeepingInterval,
-		"housekeeping-interval",
-		server.DefaultHousekeepingInterval,
-		"housekeeping interval between housekeeping runs",
+		&housekeepingIntervalDeactivateCandidates,
+		"housekeeping-interval-Deactivate-Candidates",
+		server.DefaultHousekeepingIntervalDeactivateCandidates,
+		"housekeeping Interval deactivate candidates between housekeeping runs",
+	)
+	cmd.Flags().DurationVar(
+		&housekeepingIntervalDeleteDocuments,
+		"housekeeping-interval-Delete-Documents",
+		server.DefaultHousekeepingIntervalDeleteDocuments,
+		"housekeeping Interval delete documents between housekeeping runs",
+	)
+	cmd.Flags().DurationVar(
+		&documentHardDeletionGracefulPeriod,
+		"housekeeping-DocumentHardDeletion-delete-graceful-period",
+		server.DefaultHousekeepingDocumentHardDeletionGracefulPeriod,
+		"Document deletion over time after a single housekeeping run",
 	)
 	cmd.Flags().IntVar(
-		&conf.Housekeeping.CandidatesLimitPerProject,
+		&conf.Housekeeping.ClientDeactivationCandidateLimitPerProject,
 		"housekeeping-candidates-limit-per-project",
-		server.DefaultHousekeepingCandidatesLimitPerProject,
+		server.DefaultHousekeepingClientDeactivationCandidateLimitPerProject,
 		"candidates limit per project for a single housekeeping run",
 	)
+	cmd.Flags().IntVar(
+		&conf.Housekeeping.DocumentHardDeletionCandidateLimitPerProject,
+		"housekeeping-DocumentHardDeletion-limit-per-project",
+		server.DefaultHousekeepingDocumentHardDeletionCandidateLimitPerProject,
+		"Document Deletion limit per project for a single housekeeping run",
+	)
 	cmd.Flags().IntVar(
 		&conf.Housekeeping.ProjectFetchSize,
 		"housekeeping-project-fetch-size",

server/backend/database/database.go

@@ -20,6 +20,7 @@ package database
 import (
 	"context"
 	"errors"
+	gotime "time"
 
 	"github.com/yorkie-team/yorkie/api/types"
 	"github.com/yorkie-team/yorkie/pkg/document"
@@ -157,6 +158,20 @@ type Database interface {
 		candidatesLimit int,
 	) ([]*ClientInfo, error)
 
+	// FindDocumentHardDeletionCandidatesPerProject finds the documents that need to be deleted housekeeping per project.
+	FindDocumentHardDeletionCandidatesPerProject(
+		ctx context.Context,
+		project *ProjectInfo,
+		candidatesLimit int,
+		documentHardDeletionGracefulPeriod gotime.Duration,
+	) ([]*DocInfo, error)
+
+	// DeleteDocument Document complete deletion in progress
+	DeleteDocument(
+		ctx context.Context,
+		candidates []*DocInfo,
+	) (int64, error)
+
 	// FindDocInfoByKey finds the document of the given key.
 	FindDocInfoByKey(
 		ctx context.Context,

server/backend/database/memory/database.go

@@ -619,6 +619,44 @@ func (d *DB) UpdateClientInfoAfterPushPull(
 	return nil
 }
 
+// FindDocumentHardDeletionCandidatesPerProject finds the documents that need housekeeping per project.
+func (d *DB) FindDocumentHardDeletionCandidatesPerProject(
+	_ context.Context,
+	project *database.ProjectInfo,
+	candidatesLimit int,
+	documentHardDeletionGracefulPeriod gotime.Duration,
+) ([]*database.DocInfo, error) {
+
+	txn := d.db.Txn(false)
+	defer txn.Abort()
+
+	var documents []*database.DocInfo
+	iterator, err := txn.Get(
+		tblDocuments,
+		"project_id",
+		project.ID.String(),
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf("fetch hard deletion candidates: %w", err)
+	}
+
+	currentTime := gotime.Now()
+	conditionDocumentHardDeletionGracefulPeriod := currentTime.Add(-documentHardDeletionGracefulPeriod)
+	for raw := iterator.Next(); raw != nil; raw = iterator.Next() {
+		document := raw.(*database.DocInfo)
+		if candidatesLimit <= len(documents) && candidatesLimit != 0 {
+			break
+		}
+
+		if !document.RemovedAt.After(conditionDocumentHardDeletionGracefulPeriod) {
+			documents = append(documents, document)
+		}
+	}
+
+	return documents, nil
+}
+
 // FindDeactivateCandidatesPerProject finds the clients that need housekeeping per project.
 func (d *DB) FindDeactivateCandidatesPerProject(
 	_ context.Context,
@@ -663,6 +701,30 @@ func (d *DB) FindDeactivateCandidatesPerProject(
 	return infos, nil
 }
 
+// DeleteDocument Deletes the documents completely.
+func (d *DB) DeleteDocument(
+	_ context.Context,
+	candidates []*database.DocInfo,
+) (int64, error) {
+	if len(candidates) <= 0 {
+		return 0, nil
+	}
+
+	txn := d.db.Txn(true)
+	defer txn.Abort()
+
+	var deletedCount int64
+	for _, candidate := range candidates {
+		if err := txn.Delete(tblDocuments, candidate); err != nil {
+			return 0, fmt.Errorf("fetch hard deletion candidates: %w", err)
+		}
+		deletedCount++
+	}
+	txn.Commit()
+
+	return deletedCount, nil
+}
+
 // FindDocInfoByKeyAndOwner finds the document of the given key. If the
 // createDocIfNotExist condition is true, create the document if it does not
 // exist.

server/backend/database/memory/database_test.go

@@ -103,4 +103,8 @@ func TestDB(t *testing.T) {
 	t.Run("IsDocumentAttached test", func(t *testing.T) {
 		testcases.RunIsDocumentAttachedTest(t, db, projectID)
 	})
+
+	t.Run("DocumentHardDeletion test", func(t *testing.T) {
+		testcases.RunDocumentHardDeletionTest(t, db)
+	})
 }

server/backend/database/memory/indexes.go

@@ -117,6 +117,11 @@ var schema = &memdb.DBSchema{
 					Unique:  true,
 					Indexer: &memdb.StringFieldIndex{Field: "ID"},
 				},
+				"project_id": {
+					Name:    "project_id",
+					Unique:  true,
+					Indexer: &memdb.StringFieldIndex{Field: "ProjectID"},
+				},
 				"project_id_id": {
 					Name:   "project_id_id",
 					Unique: true,

server/backend/database/mongo/client.go

@@ -227,6 +227,35 @@ func (c *Client) CreateProjectInfo(
 	return info, nil
 }
 
+// DeleteDocument Deletes the documents completely.
+func (c *Client) DeleteDocument(
+	ctx context.Context,
+	candidates []*database.DocInfo,
+) (int64, error) {
+	if len(candidates) <= 0 {
+		return 0, nil
+	}
+
+	var idList []types.ID
+	for _, docInfo := range candidates {
+		idList = append(
+			idList,
+			docInfo.ID,
+		)
+	}
+
+	deletedResult, err := c.collection(ColDocuments).DeleteMany(
+		ctx,
+		bson.M{"_id": bson.M{"$in": idList}},
+	)
+
+	if err != nil {
+		return deletedResult.DeletedCount, fmt.Errorf("deletion Error : %w", err)
+	}
+
+	return deletedResult.DeletedCount, nil
+}
+
 // FindNextNCyclingProjectInfos finds the next N cycling projects from the given projectID.
 func (c *Client) FindNextNCyclingProjectInfos(
 	ctx context.Context,
@@ -619,6 +648,34 @@ func (c *Client) UpdateClientInfoAfterPushPull(
 	return nil
 }
 
+// FindDocumentHardDeletionCandidatesPerProject finds the documents that need housekeeping per project.
+func (c *Client) FindDocumentHardDeletionCandidatesPerProject(
+	ctx context.Context,
+	project *database.ProjectInfo,
+	candidatesLimit int,
+	documentHardDeletionGracefulPeriod gotime.Duration,
+) ([]*database.DocInfo, error) {
+
+	currentTime := gotime.Now()
+	hardDeletionGracefulPeriod := currentTime.Add(-documentHardDeletionGracefulPeriod)
+
+	var DocInfos []*database.DocInfo
+	cursor, err := c.collection(ColDocuments).Find(ctx, bson.M{
+		"project_id": project.ID,
+		"removed_at": bson.M{"$lte": hardDeletionGracefulPeriod},
+	}, options.Find().SetLimit(int64(candidatesLimit)))
+
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cursor.All(ctx, &DocInfos); err != nil {
+		return nil, fmt.Errorf("fetch hard deletion candidates: %w", err)
+	}
+
+	return DocInfos, nil
+}
+
 // FindDeactivateCandidatesPerProject finds the clients that need housekeeping per project.
 func (c *Client) FindDeactivateCandidatesPerProject(
 	ctx context.Context,
@@ -650,6 +707,37 @@ func (c *Client) FindDeactivateCandidatesPerProject(
 	return clientInfos, nil
 }
 
+// FindDeactivateCandidates finds the clients that need housekeeping.
+func (c *Client) FindDeactivateCandidates(
+	ctx context.Context,
+	documentHardDeletionCandidateLimitPerProject int,
+	projectFetchSize int,
+	lastProjectID types.ID,
+) (types.ID, []*database.ClientInfo, error) {
+	projects, err := c.FindNextNCyclingProjectInfos(ctx, projectFetchSize, lastProjectID)
+	if err != nil {
+		return database.DefaultProjectID, nil, err
+	}
+
+	var candidates []*database.ClientInfo
+	for _, project := range projects {
+		clientInfos, err := c.FindDeactivateCandidatesPerProject(ctx, project, documentHardDeletionCandidateLimitPerProject)
+		if err != nil {
+			return database.DefaultProjectID, nil, err
+		}
+
+		candidates = append(candidates, clientInfos...)
+	}
+
+	var topProjectID types.ID
+	if len(projects) < projectFetchSize {
+		topProjectID = database.DefaultProjectID
+	} else {
+		topProjectID = projects[len(projects)-1].ID
+	}
+	return topProjectID, candidates, nil
+}
+
 // FindDocInfoByKeyAndOwner finds the document of the given key. If the
 // createDocIfNotExist condition is true, create the document if it does not
 // exist.

server/backend/database/mongo/client_test.go

@@ -120,4 +120,8 @@ func TestClient(t *testing.T) {
 	t.Run("IsDocumentAttached test", func(t *testing.T) {
 		testcases.RunIsDocumentAttachedTest(t, cli, dummyProjectID)
 	})
+
+	t.Run("DocumentHardDeletion test", func(t *testing.T) {
+		testcases.RunDocumentHardDeletionTest(t, cli)
+	})
 }

server/backend/database/testcases/testcases.go

@@ -1214,3 +1214,54 @@ func AssertKeys(t *testing.T, expectedKeys []key.Key, infos []*database.DocInfo)
 	}
 	assert.EqualValues(t, expectedKeys, keys)
 }
+
+// RunDocumentHardDeletionTest runs Delete document permanently
+func RunDocumentHardDeletionTest(t *testing.T, db database.Database) {
+	t.Run("housekeeping DocumentHardDeletion test", func(t *testing.T) {
+		ctx := context.Background()
+		projectInfo, err := db.CreateProjectInfo(ctx, t.Name(), dummyOwnerID, clientDeactivateThreshold)
+		assert.NoError(t, err)
+		projectID := projectInfo.ID
+
+		// Create a client and two documents
+		clientInfo, err := db.ActivateClient(ctx, projectID, t.Name())
+		assert.NoError(t, err)
+		docInfo, err := db.FindDocInfoByKeyAndOwner(ctx, clientInfo.RefKey(), helper.TestDocKey(t), true)
+		assert.NoError(t, err)
+		assert.NoError(t, clientInfo.AttachDocument(docInfo.ID))
+		assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo))
+
+		assert.NoError(t, clientInfo.RemoveDocument(docInfo.ID))
+
+		doc := document.New(key.Key(t.Name()))
+		pack := doc.CreateChangePack()
+		err = db.CreateChangeInfos(ctx, projectID, docInfo, 0, pack.Changes, true)
+		assert.NoError(t, err)
+
+		fetchSize := 100
+		lastProjectID := database.DefaultProjectID
+
+		var candidates []*database.DocInfo
+		GracePeriod := "0s"
+		documentHardDeletionGracefulPeriod, err := gotime.ParseDuration(GracePeriod)
+		assert.NoError(t, err)
+
+		candidates, err = db.FindDocumentHardDeletionCandidatesPerProject(
+			ctx,
+			projectInfo,
+			fetchSize,
+			documentHardDeletionGracefulPeriod,
+		)
+		assert.NoError(t, err)
+		assert.Equal(t, database.DefaultProjectID, lastProjectID)
+
+		deletedDocumentsCount, err := db.DeleteDocument(ctx, candidates)
+		assert.NoError(t, err)
+		assert.Equal(t, int(deletedDocumentsCount), len(candidates))
+
+		_, err = db.FindDocInfoByRefKey(ctx, docInfo.RefKey())
+		assert.ErrorIs(t, err, database.ErrDocumentNotFound)
+
+	})
+
+}