Skip to content

Commit

Permalink
Merge pull request vmware-tanzu#5233 from Lyndon-Li/udmrepo-dev-06
Browse files Browse the repository at this point in the history 
Kopia Integration:Kopia Lib - Implementation
  • Loading branch information
@qiuming-best
qiuming-best authored Aug 28, 2022
2 parents 86762f4 + 3769cd2 commit 3e30a3d
Show file tree
Hide file tree
Showing 18 changed files with 2,617 additions and 174 deletions.
2 changes: 2 additions & 0 deletions changelogs/unreleased/5233-lyndon
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Add changes for Kopia Integration: Kopia Lib - method implementation
Add changes to write Kopia Repository logs to Velero log
204 changes: 109 additions & 95 deletions design/unified-repo-and-kopia-integration/unified-repo-and-kopia-integration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,126 +93,140 @@ Velero by default uses the Unified Repository for all kinds of data movement, it
## The Unified Repository Interface
Below are the definitions of the Unified Repository Interface. All the functions are synchronization functions.
```
///BackupRepoService is used to initialize, open or maintain a backup repository
// BackupRepoService is used to initialize, open or maintain a backup repository
type BackupRepoService interface {
///Create a backup repository or connect to an existing backup repository
///repoOption: option to the backup repository and the underlying backup storage
///createNew: indicates whether to create a new or connect to an existing backup repository
///result: the backup repository specific output that could be used to open the backup repository later
Init(ctx context.Context, repoOption RepoOptions, createNew bool) error
///Open an backup repository that has been created/connected
///repoOption: options to open the backup repository and the underlying storage
Open(ctx context.Context, repoOption RepoOptions) (BackupRepo, error)
///Periodically called to maintain the backup repository to eliminate redundant data and improve performance
///repoOption: options to maintain the backup repository
Maintain(ctx context.Context, repoOption RepoOptions) error
// Init creates a backup repository or connect to an existing backup repository.
// repoOption: option to the backup repository and the underlying backup storage.
// createNew: indicates whether to create a new or connect to an existing backup repository.
Init(ctx context.Context, repoOption RepoOptions, createNew bool) error
// Open opens an backup repository that has been created/connected.
// repoOption: options to open the backup repository and the underlying storage.
Open(ctx context.Context, repoOption RepoOptions) (BackupRepo, error)
// Maintain is periodically called to maintain the backup repository to eliminate redundant data.
// repoOption: options to maintain the backup repository.
Maintain(ctx context.Context, repoOption RepoOptions) error
// DefaultMaintenanceFrequency returns the defgault frequency of maintenance, callers refer this
// frequency to maintain the backup repository to get the best maintenance performance
DefaultMaintenanceFrequency() time.Duration
}
///BackupRepo provides the access to the backup repository
// BackupRepo provides the access to the backup repository
type BackupRepo interface {
///Open an existing object for read
///id: the object's unified identifier
OpenObject(ctx context.Context, id ID) (ObjectReader, error)
///Get a manifest data
GetManifest(ctx context.Context, id ID, mani *RepoManifest) error
///Get one or more manifest data that match the given labels
FindManifests(ctx context.Context, filter ManifestFilter) ([]*ManifestEntryMetadata, error)
///Create a new object and return the object's writer interface
///return: A unified identifier of the object on success
NewObjectWriter(ctx context.Context, opt ObjectWriteOptions) ObjectWriter
///Save a manifest object
PutManifest(ctx context.Context, mani RepoManifest) (ID, error)
///Delete a manifest object
DeleteManifest(ctx context.Context, id ID) error
///Flush all the backup repository data
Flush(ctx context.Context) error
///Get the local time of the backup repository. It may be different from the time of the caller
Time() time.Time
///Close the backup repository
Close(ctx context.Context) error
}
// OpenObject opens an existing object for read.
// id: the object's unified identifier.
OpenObject(ctx context.Context, id ID) (ObjectReader, error)
// GetManifest gets a manifest data from the backup repository.
GetManifest(ctx context.Context, id ID, mani *RepoManifest) error
// FindManifests gets one or more manifest data that match the given labels
FindManifests(ctx context.Context, filter ManifestFilter) ([]*ManifestEntryMetadata, error)
// NewObjectWriter creates a new object and return the object's writer interface.
// return: A unified identifier of the object on success.
NewObjectWriter(ctx context.Context, opt ObjectWriteOptions) ObjectWriter
// PutManifest saves a manifest object into the backup repository.
PutManifest(ctx context.Context, mani RepoManifest) (ID, error)
// DeleteManifest deletes a manifest object from the backup repository.
DeleteManifest(ctx context.Context, id ID) error
// Flush flushes all the backup repository data
Flush(ctx context.Context) error
// Time returns the local time of the backup repository. It may be different from the time of the caller
Time() time.Time
// Close closes the backup repository
Close(ctx context.Context) error
type ObjectReader interface {
io.ReadCloser
io.Seeker
///Length returns the logical size of the object
Length() int64
io.ReadCloser
io.Seeker
// Length returns the logical size of the object
Length() int64
}
type ObjectWriter interface {
io.WriteCloser
///For some cases, i.e. block incremental, the object is not written sequentially
io.Seeker
// Periodically called to preserve the state of data written to the repo so far
// Return a unified identifier that represent the current state
// An empty ID could be returned on success if the backup repository doesn't support this
Checkpoint() (ID, error)
///Wait for the completion of the object write
///Result returns the object's unified identifier after the write completes
Result() (ID, error)
}
io.WriteCloser
// Seeker is used in the cases that the object is not written sequentially
io.Seeker
// Checkpoint is periodically called to preserve the state of data written to the repo so far.
// Checkpoint returns a unified identifier that represent the current state.
// An empty ID could be returned on success if the backup repository doesn't support this.
Checkpoint() (ID, error)
// Result waits for the completion of the object write.
// Result returns the object's unified identifier after the write completes.
Result() (ID, error)
}
```

Some data structure & constants used by the interfaces:
```
```
type RepoOptions struct {
///A repository specific string to identify a backup storage, i.e., "s3", "filesystem"
StorageType string
///Backup repository password, if any
RepoPassword string
///A custom path to save the repository's configuration, if any
ConfigFilePath string
///Other repository specific options
GeneralOptions map[string]string
///Storage specific options
StorageOptions map[string]string
// StorageType is a repository specific string to identify a backup storage, i.e., "s3", "filesystem"
StorageType string
// RepoPassword is the backup repository's password, if any
RepoPassword string
// ConfigFilePath is a custom path to save the repository's configuration, if any
ConfigFilePath string
// GeneralOptions takes other repository specific options
GeneralOptions map[string]string
// StorageOptions takes storage specific options
StorageOptions map[string]string
// Description is a description of the backup repository/backup repository operation.
// It is for logging/debugging purpose only and doesn't control any behavior of the backup repository.
Description string
}
///ObjectWriteOptions defines the options when creating an object for write
// ObjectWriteOptions defines the options when creating an object for write
type ObjectWriteOptions struct {
FullPath string ///Full logical path of the object
Description string ///A description of the object, could be empty
Prefix ID ///A prefix of the name used to save the object
AccessMode int ///OBJECT_DATA_ACCESS_*
BackupMode int ///OBJECT_DATA_BACKUP_*
FullPath string // Full logical path of the object
DataType int // OBJECT_DATA_TYPE_*
Description string // A description of the object, could be empty
Prefix ID // A prefix of the name used to save the object
AccessMode int // OBJECT_DATA_ACCESS_*
BackupMode int // OBJECT_DATA_BACKUP_*
}
const (
///Below consts defines the access mode when creating an object for write
OBJECT_DATA_ACCESS_MODE_UNKNOWN int = 0
OBJECT_DATA_ACCESS_MODE_FILE int = 1
OBJECT_DATA_ACCESS_MODE_BLOCK int = 2
OBJECT_DATA_BACKUP_MODE_UNKNOWN int = 0
OBJECT_DATA_BACKUP_MODE_FULL int = 1
OBJECT_DATA_BACKUP_MODE_INC int = 2
// Below consts descrbe the data type of one object.
// Metadata: This type describes how the data is organized.
// For a file system backup, the Metadata describes a Dir or File.
// For a block backup, the Metadata describes a Disk and its incremental link.
ObjectDataTypeUnknown int = 0
ObjectDataTypeMetadata int = 1
ObjectDataTypeData int = 2
// Below consts defines the access mode when creating an object for write
ObjectDataAccessModeUnknown int = 0
ObjectDataAccessModeFile int = 1
ObjectDataAccessModeBlock int = 2
ObjectDataBackupModeUnknown int = 0
ObjectDataBackupModeFull int = 1
ObjectDataBackupModeInc int = 2
)
///ManifestEntryMetadata is the metadata describing one manifest data
// ManifestEntryMetadata is the metadata describing one manifest data
type ManifestEntryMetadata struct {
ID ID ///The ID of the manifest data
Length int32 ///The data size of the manifest data
Labels map[string]string ///Labels saved together with the manifest data
ModTime time.Time ///Modified time of the manifest data
ID ID // The ID of the manifest data
Length int32 // The data size of the manifest data
Labels map[string]string // Labels saved together with the manifest data
ModTime time.Time // Modified time of the manifest data
}
type RepoManifest struct {
Payload interface{} ///The user data of manifest
Metadata *ManifestEntryMetadata ///The metadata data of manifest
Payload interface{} // The user data of manifest
Metadata *ManifestEntryMetadata // The metadata data of manifest
}
type ManifestFilter struct {
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ require (
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-logr/logr v0.4.0 // indirect
github.com/go-logr/zapr v0.4.0 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
Expand Down
1 change: 1 addition & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,7 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
Expand Down
25 changes: 13 additions & 12 deletions pkg/repository/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package provider

import (
"context"
"time"

velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)
Expand All @@ -30,27 +31,27 @@ type RepoParam struct {

// Provider defines the methods to manipulate a backup repository
type Provider interface {
//InitRepo is to initialize a repository from a new storage place
// InitRepo is to initialize a repository from a new storage place
InitRepo(ctx context.Context, param RepoParam) error

//ConnectToRepo is to establish the connection to a
//storage place that a repository is already initialized
// ConnectToRepo is to establish the connection to a
// storage place that a repository is already initialized
ConnectToRepo(ctx context.Context, param RepoParam) error

//PrepareRepo is a combination of InitRepo and ConnectToRepo,
//it may do initializing + connecting, connecting only if the repository
//is already initialized, or do nothing if the repository is already connected
// PrepareRepo is a combination of InitRepo and ConnectToRepo,
// it may do initializing + connecting, connecting only if the repository
// is already initialized, or do nothing if the repository is already connected
PrepareRepo(ctx context.Context, param RepoParam) error

//PruneRepo does a full prune/maintenance of the repository
// PruneRepo does a full prune/maintenance of the repository
PruneRepo(ctx context.Context, param RepoParam) error

//PruneRepoQuick does a quick prune/maintenance of the repository if available
PruneRepoQuick(ctx context.Context, param RepoParam) error

//EnsureUnlockRepo esures to remove any stale file locks in the storage
// EnsureUnlockRepo esures to remove any stale file locks in the storage
EnsureUnlockRepo(ctx context.Context, param RepoParam) error

//Forget is to delete a snapshot from the repository
// Forget is to delete a snapshot from the repository
Forget(ctx context.Context, snapshotID string, param RepoParam) error

// DefaultMaintenanceFrequency returns the default frequency to run maintenance
DefaultMaintenanceFrequency(ctx context.Context, param RepoParam) time.Duration
}
10 changes: 5 additions & 5 deletions pkg/repository/provider/restic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package provider

import (
"context"
"time"

"github.com/sirupsen/logrus"

Expand Down Expand Up @@ -55,15 +56,14 @@ func (r *resticRepositoryProvider) PruneRepo(ctx context.Context, param RepoPara
return r.svc.PruneRepo(param.BackupLocation, param.BackupRepo)
}

func (r *resticRepositoryProvider) PruneRepoQuick(ctx context.Context, param RepoParam) error {
// restic doesn't support this operation
return nil
}

func (r *resticRepositoryProvider) EnsureUnlockRepo(ctx context.Context, param RepoParam) error {
return r.svc.UnlockRepo(param.BackupLocation, param.BackupRepo)
}

func (r *resticRepositoryProvider) Forget(ctx context.Context, snapshotID string, param RepoParam) error {
return r.svc.Forget(param.BackupLocation, param.BackupRepo, snapshotID)
}

func (r *resticRepositoryProvider) DefaultMaintenanceFrequency(ctx context.Context, param RepoParam) time.Duration {
return r.svc.DefaultMaintenanceFrequency()
}
Loading

0 comments on commit 3e30a3d

Please sign in to comment.