From c3d1d83da50ad9493c8eb0b94e16d7fc88f28387 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 24 Jan 2023 12:59:53 -0500 Subject: [PATCH] BIAv2 async operations controller work Signed-off-by: Scott Seago --- changelogs/unreleased/5849-sseago | 1 + config/crd/v1/bases/velero.io_backups.yaml | 21 + config/crd/v1/bases/velero.io_schedules.yaml | 5 + config/crd/v1/crds/crds.go | 4 +- design/biav2-design.md | 14 + design/general-progress-monitoring.md | 211 +++++--- internal/hook/item_hook_handler.go | 14 + pkg/apis/velero/v1/backup.go | 39 +- pkg/apis/velero/v1/zz_generated.deepcopy.go | 1 + pkg/archive/filesystem.go | 19 +- pkg/archive/filesystem_test.go | 18 + pkg/backup/backup.go | 198 +++++++ pkg/backup/backup_test.go | 236 ++++++++- pkg/backup/item_backupper.go | 208 ++++---- pkg/backup/item_collector.go | 64 ++- pkg/backup/request.go | 11 + pkg/builder/backup_builder.go | 6 + pkg/cmd/cli/backup/create.go | 5 +- pkg/cmd/cli/backup/create_test.go | 2 + pkg/cmd/cli/backup/logs.go | 4 +- pkg/cmd/cli/schedule/create.go | 1 + pkg/cmd/server/server.go | 75 ++- pkg/cmd/server/server_test.go | 12 +- pkg/cmd/util/output/backup_describer.go | 67 +++ .../async_backup_operations_controller.go | 483 ++++++++++++++++++ ...async_backup_operations_controller_test.go | 308 +++++++++++ pkg/controller/backup_controller.go | 197 ++++--- pkg/controller/backup_controller_test.go | 118 ++--- pkg/controller/backup_finalizer_controller.go | 204 ++++++++ .../backup_finalizer_controller_test.go | 184 +++++++ pkg/controller/backup_sync_controller.go | 20 + pkg/controller/backup_sync_controller_test.go | 188 +++++-- pkg/controller/constants.go | 4 + pkg/controller/download_request_controller.go | 22 +- .../download_request_controller_test.go | 1 + pkg/itemoperation/backup_operation.go | 45 +- pkg/itemoperation/restore_operation.go | 6 +- pkg/itemoperation/shared.go | 38 +- pkg/persistence/mocks/backup_store.go | 208 ++++++-- pkg/persistence/object_store.go | 10 + pkg/persistence/object_store_test.go | 17 +- .../v2/restartable_backup_item_action.go | 18 +- .../v2/restartable_backup_item_action_test.go | 4 +- .../v2/backup_item_action_client.go | 33 +- .../v2/backup_item_action_server.go | 11 +- .../v2/backup_item_action_test.go | 3 +- .../v2/BackupItemAction.pb.go | 155 +++--- .../v2/BackupItemAction.proto | 1 + .../backupitemaction/v2/backup_item_action.go | 15 +- .../backupitemaction/v2/BackupItemAction.go | 50 +- pkg/plugin/velero/shared.go | 14 + site/content/docs/main/api-types/backup.md | 13 +- 52 files changed, 3060 insertions(+), 546 deletions(-) create mode 100644 changelogs/unreleased/5849-sseago create mode 100644 pkg/controller/async_backup_operations_controller.go create mode 100644 pkg/controller/async_backup_operations_controller_test.go create mode 100644 pkg/controller/backup_finalizer_controller.go create mode 100644 pkg/controller/backup_finalizer_controller_test.go diff --git a/changelogs/unreleased/5849-sseago b/changelogs/unreleased/5849-sseago new file mode 100644 index 0000000000..4e64ab652f --- /dev/null +++ b/changelogs/unreleased/5849-sseago @@ -0,0 +1 @@ +BIAv2 async operations controller work diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 2fb76533a1..1cfa693145 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -273,6 +273,11 @@ spec: type: string nullable: true type: array + itemOperationTimeout: + description: ItemOperationTimeout specifies the time used to wait + for asynchronous BackupItemAction operations The default value is + 1 hour. + type: string labelSelector: description: LabelSelector is a metav1.LabelSelector to filter with when adding individual objects to the backup. If empty or nil, all @@ -415,6 +420,20 @@ spec: status: description: BackupStatus captures the current status of a Velero backup. properties: + asyncBackupItemOperationsAttempted: + description: AsyncBackupItemOperationsAttempted is the total number + of attempted async BackupItemAction operations for this backup. + type: integer + asyncBackupItemOperationsCompleted: + description: AsyncBackupItemOperationsCompleted is the total number + of successfully completed async BackupItemAction operations for + this backup. + type: integer + asyncBackupItemOperationsFailed: + description: AsyncBackupItemOperationsFailed is the total number of + async BackupItemAction operations for this backup which ended with + an error. + type: integer completionTimestamp: description: CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time @@ -457,6 +476,8 @@ spec: - InProgress - WaitingForPluginOperations - WaitingForPluginOperationsPartiallyFailed + - FinalizingAfterPluginOperations + - FinalizingAfterPluginOperationsPartiallyFailed - Completed - PartiallyFailed - Failed diff --git a/config/crd/v1/bases/velero.io_schedules.yaml b/config/crd/v1/bases/velero.io_schedules.yaml index 9e7454d517..22405ad12d 100644 --- a/config/crd/v1/bases/velero.io_schedules.yaml +++ b/config/crd/v1/bases/velero.io_schedules.yaml @@ -308,6 +308,11 @@ spec: type: string nullable: true type: array + itemOperationTimeout: + description: ItemOperationTimeout specifies the time used to wait + for asynchronous BackupItemAction operations The default value + is 1 hour. + type: string labelSelector: description: LabelSelector is a metav1.LabelSelector to filter with when adding individual objects to the backup. If empty diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 3809146e70..e89df169e4 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -30,14 +30,14 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WAo\xdc6\x13\xbd\xebW\f\xf2\x1dr\xf9\xa4M\xd0C\v\xddR\xb7\x05\x82&\x86a\a\xbe\x14=P\xe4\xec.c\x8ad\xc9\xe1\xa6ۢ\xff\xbd\x18R\xf2j%\xd9\x1b\a\xa8n\"\x87of\xde\xcc\x1bQU]ו\xf0\xfa\x1eC\xd4ζ \xbc\xc6?\t-\xbf\xc5\xe6\xe1\x87\xd8h\xb79\xbc\xad\x1e\xb4U-\\\xa5H\xae\xbf\xc5\xe8R\x90\xf8\x13n\xb5դ\x9d\xadz$\xa1\x04\x89\xb6\x02\x10\xd6:\x12\xbc\x1c\xf9\x15@:K\xc1\x19\x83\xa1ޡm\x1eR\x87]\xd2Fa\xc8\xe0\xa3\xebÛ\xe6\xfb\xe6M\x05 \x03\xe6\xe3\x9ft\x8f\x91D\xef[\xb0ɘ\n\xc0\x8a\x1e[\xe8\x84|H>\xa0wQ\x93\v\x1acs@\x83\xc15\xdaUѣd\xb7\xbb\xe0\x92o\xe1\xb4QN\x0f!\x95t~\xcc@\xb7#\xd01o\x19\x1d\xe9\xd7\xd5\xed\x0f:R6\xf1&\x05a\xd6\x02\xc9\xdbQ\xdb]2\",\f\xd8A\x94\xcec\v\xd7\x1c\x8b\x17\x12U\x050P\x90c\xabA(\x95I\x15\xe6&hK\x18\xae\x9cI\xfdHf\r\x9f\xa3\xb37\x82\xf6-4#\xed͂\xb2l;\x12\xf6n\x87\xc3;\x1dٹ\x12\x84K0f\xae9\xc5\xfa\xe9\xe8\xf1\f\xe5D\x04L\xf6\nb\xa4\xa0\xed\xae:\x19\x1f\xde\x16*\xe4\x1e{\xd1\x0e\xb6Σ}w\xf3\xfe\xfe\xbb\xbb\xb3e\x00\x1f\x9c\xc7@z,Oy&}9Y\x05P\x18eОr\u05fcf\xc0b\x05\x8a\x1b\x12#\xd0\x1eGNQ\r1\x80\xdb\x02\xedu\x84\x80>`D[Z\xf4\f\x18\xd8HXp\xddg\x94\xd4\xc0\x1d\x06\x86\x81\xb8w\xc9(\xee\xe3\x03\x06\x82\x80\xd2\xed\xac\xfe\xeb\x11;\x02\xb9\xec\xd4\b¡GNO\xae\xa1\x15\x06\x0e\xc2$\xfc?\b\xab\xa0\x17G\b\xc8^ \xd9\t^6\x89\r|t\x01AۭkaO\xe4c\xbb\xd9\xec4\x8dz\x94\xae\xef\x93\xd5t\xdcdi\xe9.\x91\vq\xa3\xf0\x80f\x13\xf5\xae\x16A\xee5\xa1\xa4\x14p#\xbc\xaes\xe86k\xb2\xe9\xd5\xff\u00a0\xe0\xf8\xfa,\xd6E-˓\xc5\xf2L\x05X-\xa0#\x88\xe1h\xc9\xe2D4/1;\xb7?\xdf}\x82\xd1u.Ɯ\xfd\xcc\xfb\xe9`<\x95\x80\t\xd3v\x8b\xa1\x14q\x1b\\\x9f1\xd1*ﴥ\xfc\"\x8dF;\xa7?\xa6\xae\xd7\xc4u\xff#a$\xaeU\x03WyHA\x87\x90<\xabA5\xf0\xde\u0095\xe8\xd1\\\x89\x88\xffy\x01\x98\xe9X3\xb1_W\x82\xe9|\x9d\x1b\x17\xd6&\x1b\xe3\b|\xa2^\xf3\xb1v\xe7Qr\xf9\x98A>\xaa\xb7Zfm\xc0\xd6\x05\x10\v\xfb\xe6\fz]\xba\xfc\x94\xe1wG.\x88\x1d~p\x05sn\xb4\x1a\xdb\xec\xcc\x18\x1cO\x96\"c\\7\\`\x03\xd0^\xd0D\xbf$\xb4}\x1c\x03\xab\xf90\x92-S\bhi\x80\xc97\x88o\x1e\x99FD\x9a\x8c\v\xbe\xcd]\xe8\x80\x0f\xcb\x13c`\f\x06\xc4\v\xd3\xf9\xf2E̿\xba\xb9hk\x93e\xebB/\xa8\\\x17k\x06ZX\xf0\xb5\\t\x06[\xa0\x90\x96\xdb\xcf\xcdQ\x8cQ\xec.e\xf7\xb1X\x95\xcb\xc5p\x04D\xe7\x12=A=\xed\x97Q\xc0\x85r\\\x88\xd4\xefE\xbc\x14\xe7\r۬5\xc4\xec{\xf5\\\bO\xcd\xcck\xfc\xb2\xb2z\x8bB-u\\õ\xa3\xf5\xad'3\\U\xc5b1\xf2=LM\xea\x1c\x8b\x90\xa7+\xa9{\xbcW\xb6\xf0\xf7?\xd5IXBJ\xf4\x84\xeaz\xfe\a6\xcc\xf7\xf1\x87*\xbfJg\xcb\x0fPl\xe1\xb7߫\xe2\n\xd5\xfd\xf8\x93ċ\xff\x06\x00\x00\xff\xff\xc8p\x98۸\x0e\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Ms\xdc:rw\xfd\x8a.\xe5\xe0\xdd*\xcd\xe8\xb9rHJ7?Y\xae\xa8ދ\xad\xb2\xb4\xdaC\x92\x03\x86\xec\x99\xc1\x13\bp\x01p\xe4I*\xff=\xd5\x00\xf8\r\x92\x18Y\xda}/e\\l\x91@\x03\xe8n\xf4\x17\x9a=g\xab\xd5ꌕ\xfc\x11\xb5\xe1J^\x01+9~\xb3(\xe9/\xb3~\xfaW\xb3\xe6\xea\xf2\xf0\xfe\xec\x89\xcb\xfc\n\xae+cU\xf1\x15\x8d\xaat\x86\x1fq\xcb%\xb7\\ɳ\x02-˙eWg\x00LJe\x19=6\xf4'@\xa6\xa4\xd5J\bԫ\x1d\xca\xf5S\xb5\xc1M\xc5E\x8e\xda\x01\xaf\xa7>\xfc\xb4\xfe\x97\xf5Og\x00\x99F7\xfc\x81\x17h,+\xca+\x90\x95\x10g\x00\x92\x15x\x05\x1b\x96=U\xa5Y\x1fP\xa0Vk\xae\xceL\x89\x19͵Ӫ*\xaf\xa0}ᇄu\xf8=\xfc\xecF\xbb\a\x82\x1b\xfbK\xe7\xe1\xaf\xdcX\xf7\xa2\x14\x95f\xa2\x99\xc9=3\\\xee*\xc1t\xfd\xf4\f\xc0d\xaa\xc4+\xf8LS\x94,\xc3\xfc\f l\xc7M\xb9\n\v>\xbc\xf7\x10\xb2=\x16̯\x05@\x95(?\xdc\xdd>\xfe\xf3}\xef1@\x8e&Ӽ\xb4\x0e)~a\xc0\r0xt\xdb\x02\x1d\xd0\x0fv\xcf,h,5\x1a\x94ր\xdd#d\xac\xb4\x95FP[\xf8\xa5ڠ\x96h\xd14\xa0\x012Q\x19\x8b\x1a\x8ce\x16\x81Y`P*.-p\t\x96\x17\b\u007f\xfapw\vj\xf3\x1bf\xd6\x00\x9390cTƙ\xc5\x1c\x0eJT\x05\xfa\xb1\u007f^7PK\xadJԖ\xd7x\xf6\xad\xc3U\x9d\xa7\x83\xed\xbd#\f\xf8^\x90\x13;\xa1\xdfF\xc0\"\xe6\x01i\xb4\x1f\xbb\xe7\xa6ݮ\xe3\x90\x1e`\xa0NL\x86ů\xe1\x1e5\x81\x01\xb3W\x95ȉ\v\x0f\xa8\ta\x99\xdaI\xfe\xdf\rl\x03V\xb9I\x05\xb3\x18\x18\xa0m\\ZԒ\t80Q\xe1\x85CI\xc1\x8e\xa0\x91f\x81Jv\xe0\xb9.f\r\xff\xae4\x02\x97[u\x05{kKsuy\xb9\xe3\xb6>M\x99*\x8aJr{\xbct\a\x83o*\xab\xb4\xb9\xcc\xf1\x80\xe2\xd2\xf0݊\xe9l\xcf-fD\xc8KV\xf2\x95[\xbat'j]\xe4\xffT3\x80y\xd7[\xab=\x123\x1a\xab\xb9\xdcu^8\xae\x9f\xa1\x00\x1d\x00\xcf_~\xa8\xdfE\x8bhzD\xd8\xf9zs\xff\xd0\xe5=n\x86\xd8wx\xef0dK\x02B\x18\x97[Ԟ\x88[\xad\n\a\x13e\xee\xb9ϱ\xae\xe0(\x87\xe87զ\xe0\x96\xe8\xfe\xb7\n\r1\xb9Zõ\x131\xb0A\xa8ʜ8s\r\xb7\x12\xaeY\x81\xe2\x9a\x19|s\x02\x10\xa6͊\x10\x9bF\x82\xaet\x1cv\xf6X뼨e\xd9\x04\xbd\xbc@\xb8/1\xeb\x1d\x18\x1aŷ\xd8V$K\xc6x\x06 ^\x9e\xe4\x01.\x8dE\x96\xaf\xcf_\x93@\xf8-\x13U\x8eyc\xb6\x8cd\xc0\x8087\xa3\x01Τc\\\x92\xd6 #\x8a\x90+۷d\x98D\xb6\xca4\x02\xc9m.=\x97\xf3\xeeGZ.͋3h\xfa\x192\x13\"\xfa\xd4+\xa3\xf4D\xe1\xf4\x1c\x99\xf9\xa4\x96S2c\x86y/\x93@\x97\xf3aR<Džܗ\x17d\xbc$f;~\xf7\xc5XJNˋ2Y\x16\x13\x02\x13\xf3W\xfa\x99)\xf3 O\xc8ZIB\xcer\x86\xca\xc9y)!\x0fdv\x1f\xc9\xd9(\x91<\x93Y\xc0\x939(s\xd9%\vQ\xa9q\xe6IzN\xc9,h\x97o\xb2\x9cI\xf2z\xf9\xa2\xafa\x03O\x8b\x9a\xc5l\x90E\x1by~}\x8b\xf9\x1e\xa7dy,b\xec\x85\x19\x1dM\xc6\xc6ļ\xa7\xe6q\xf4\xf34&\x80\xa6doLdgL@\x9c\xcd\xd9H\xcdɘ\x80\xbd\xa0vg\xb9d\xe6e\xfcC`X\xd4o\xe2\xef\xc5Q/ݘ\xd2=sq\xc9B\xff2\xe8N\xb4\xac\xad\xa6y\xf33fyr\xbb?\xdd\xfc,*ay)\\8\xff\xc0\xf3\xa8\xd3h\xf7xl>\xeb\xfcM\xb9Ϝ6G\a\xe9\xcb׆=\xd7\x03#\x9a\x19xF!\x80Řk\xb4\xf3\xcc\u007f˞\xa9\x15\x92̧\x03\x17>X\r\x9f\xbc_x\x0ev_r\xc5\"\x9ev\x8f\x05A\xa9\xbf|=\xc1\xfd\x987\x10\xbd-\xeb\x9e\xfd\xadB}\x04u@\xddZ\f\v\xdf\x11\xf8\x83f*\xd1&n\x05\xf9\xe1+(\f\f\xe7\xf6\xc0\xc1\a\xe9UX\x14\xec`\x8d\x0e\x0e\x9dy\xd1К\xc4\x1b\xf9\x01\x13]\xe3\x81\x0fՌ\x8e\xbc_\xb2=S\x93\xf0\xdf\xd6u8\xddyXT\xdbo\xe2@\xbc܅\x98\x01\x99\x9aT\x9fv\x01\xb5\x98D\xffV\xaeĒ3\x91lE\xa5%ɿEr\xfc\tI\xf1'8\x15\xa7\xb9\x15\xc9hJI~\u007f\x13\xe7\xe2\r\u074b\xb7p0^\xe6b,\x80\x1c$\xb5\xa7\xa4\xab']\xae&\xdf/\xa4\\\x8e._\x01̧\xa1'\xa4\x9f'\\\x0e,\xad4!\xcd\xfc\xb4\xf4\xf2\x04\x1c\xbe\x91\xf3\xf1F\xee\xc7[8 o\xeb\x82,:!\x8b\x9c3\xfb\xfa\xc5\xd1e\xa5sԳ\xc1\xf8TV\x9be\xb2\x81\xbfПs\xf0Em]\xe1\x85z\xf5L\xd3XH\xb9\xf9\xfa3\x83_\xb8\xcc==\x88\xa9:z\xbcwC\xd0\x1a\x16\xf1\x04\x81\xd6j\v\x15\xb0\xfc\xb5\x82\xc1\x92iW6ms\xf4W\x93f\r7,\xdb\x0f\xa0\xef\xa3~\xc2V\xe9\x82Y8o\xeed.=p\xfa\xfb|\r\xf0I5\x97^݊\n\x86\x17\xa58\x92\x1f\x10\x81y\xde\x05\xf12\x86\x882\x93\t\xe5\x9aB\xfd\x9a\x05\xdf\xef\xbe\xdf;r\x99W\x97\xee\xa9ᚸ\xe3\xc3\xe4\x11\xee\x1e\x9du\xe2\n\x86dm\xf1\x94`\u007f\xd4\xde߰\xb6\xcaϯ\u007f\xadg\xac\xd2l\x87\xbf*_\x81k\t\a\xfd\u07bd\xf2kAj\xd4\xd7\xec\xf5W\x181m\x1aj\x81\r\x80\xb5\xd93\xa3:P\xb4ʘ8\x999\x89֊\x85\xcd<<\xfc\xea7`y\x81돕\xbfB]\x95L\x1b$l\xd6\x1b\xf3\x836\xf4߽z\x8eE9T\xd8\xf3\xcf\xc3ukt\x19:\xee\xa6\xf6\xa4\xd5\x1fz\xf5\xc4j\x14-\xb1\xe8c|T\xc7E\xeb\x10ɟ\xf6(\x87N\xc1\xe9\x94Tt\xc1\v\xf7\x85\xcd\xeb\x16\xfc\x99\x92\xdfSE\xe7\\\xa1\xb5\xe5\xb2s\xbe\x1e[(2\x19\xf2\xbc*\xed\xaa\xf5\x84Zm\xae\xba\xcd\xcb*\xcf\xf9\xb4\x94^\xe1\xcfy:]\x8fG\xb8\xf2\x8e:\xefT\x9ek\n\x80=3Ӥ\xbeDUj\vΏt6-A\xc3\x1c\xf0\x80\x12\x94t\x99.\xae\x0e\x8e/A:\x1c\x13\x81څ\x12Ri\xaaR(\x96\xd7'\xbc\xd6^\xa1l僓_\xfa\x80\xfa\x9d\x99\x81ٔt\x8b a\xacP\xbc:\xb9\x82\x9cY\\E\x81&ɾ(\xb3e\x86\xf7\x19\xdd|\xb0\x96<\x84\x98\xd5<,\x1d85\xb2\xd6\xc4VY&@V\xc5ƫvVw\x88\xd1oT@Єܧ\x99\xe3\xe57ƥ\xc5\xdd(\xba8\xde\xd9u\xcd?'\xef\xac\x199\xb53Se\x19\x1a\xb3\xad\x84\x88\x19\xf9\r\xe7\xbe\xfe6]V\xdfb\xb53\xd7ɋ@\x97\x12X\x17\xd4\xf39\x81\x05\x1a\xc3vu\x99\xb3g\xd2@;\x94\xe8\f\xa0X\xe4ѻ\x88m\x0eY\xbfȗ\x8fe\xb1\xccV,LP\xe7\x00tz\xbd\x8b\xd9MB\xed|)D^\x17\x82\xadU\xf3\x898\xf9Vr\x9d\xa2\xcao\x9a\x8e\x84\x1b\x17\x86v\x84h\v\xf7\xa2\xe0;Nz\x90\x88\xb4cz\xc3v\xb8ʔ\x10\xe8\x12\xce\xc7\xebz\xcb\xc3\x1a2\xf5\xbe\"3\x8b[\xfb\xd4\xed\x1bb\x1e\x9eھF\x06\xf3\x85\x16]\x1dW\xcb5\xb6\x85\x91G\vRn\xe2\x93T\xb7\xc7B\xb4\x84\xf0x\xa5ݾ\xf5\x01\vr5Xҡ\xa2\xf0E0\x06\xe3~m\xc1~S\xfa\x02\n.\xe9\x1f\xb2\xfb]P\xa2\x1e|\xd2\xfa]\xf5\xba\x85u\xdfQ\x9f&a\xba\xa3H\xb1>\x10S\xa6jM\x92e\x19Y\xf5xi,\x13\x11A\xfa]iWξ%6\xc7\xfc/\x11\x83o\x84\xf0\xdbn\xff\xe6[\xf5F\x8d:p\x1es.\x8d\xdd+\x91\xa8J\x05\x97܌\x12\x9e5\xb7\x96\x04w\xf7\x96\x10,\x89j!\xc0\x90\xf0\x9a\xa8L8\xa7B\xdc{R\xf2\xb7\xd3Q˾#\xd5t\x9e\xb2\x11\xc2\xe6\x14\x91e\xe3P0\xb1-\xff\x99\x147\xf5X\"e\xb6grGL\xa5U\xb5\xdb\xd7|9\xa1\x82\xa7\x82~\x15-\nJw\xb0M}Cc+-;Q\x9fpg\x93w\x96˲\xa7ɕ\x86(t]\xb3\xff2\xd4\x1a\\m\xb5*V\x81\x16\xeeb\xe5\"Db4W\xe4h\xd8}\x14\xe5\xe0\x8b!\x87\xa2^\x8e\r\xca\x12%0\x13֓\xf0\r\xd7H\bvU\xb1\x9b.\xc1\xe0\x8c\xe0\xe3O|\xeb\xaf\xf12Z\xf5\x9f\xffቾ\x87$\xe3\xe8ݬ]\xe4L\x9e\xc6\xc0Y\xa8\x81}'\x90\f\x16\x83\xd87\xb9ޝd[\x1f^\xe6-\xbe\xa6\xabX\xff\x92\xc5\xeb8P\x87\x979\x89o\xe6!\xbe\xee\ue799\xab\x9e\xbft\xc6\xfe\x1a\xbaE\\\xc4\x00!\xe2$F\xb6Ѹ\x8d\x8bNb\xc7G\xac\xd78Q`{\xe07\xbe\x92\x97\x18\xd5\x03\xa3\x87N\x80杳\x1df\nO\xda\xc8\x1b\xcb2$v\xfd<\xfcŚs_\xfe\xbd\xfeQ\x1a\xf7g\xa6\xa4W\xb7\xe6\n\xfe\xe3\xbf\xce \x84v\x1f\xeb_\x9f\xa1\x87\xff\x17\x00\x00\xff\xffp,\xdd\xe3\xddg\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c9n\xef\xfa\x15(\xe5\xc1wU\x9aѺ\xf2\x90\x94\u07bc\xb2]Q\xed\xc6VY:\xdfC\x92\aN7f\x86+6\xd9G\xb2G\x9eK忧\b\xb2\xbf\xd9ݜ\xb1t\xb7we\xbe캇\x04A\x00\x04\x01\x10\x84.V\xab\xd5\x05+\xf9WԆ+y\x03\xac\xe4\xf8͢t\xff2\xeb\xa7\u007f7k\xae\xae\x0fo/\x9e\xb8\xcco\xe0\xb62V\x15_ШJg\xf8\x1e\xb7\\r˕\xbc(в\x9cYvs\x01\xc0\xa4T\x96\xb9\xcf\xc6\xfd\x13 S\xd2j%\x04\xea\xd5\x0e\xe5\xfa\xa9\xda\xe0\xa6\xe2\"GM\xc0\xeb\xa9\x0f?\xad\xffm\xfd\xd3\x05@\xa6\x91\x86?\xf2\x02\x8deEy\x03\xb2\x12\xe2\x02@\xb2\x02o`ò\xa7\xaa4\xeb\x03\n\xd4j\xcdՅ)1ss\xed\xb4\xaa\xca\x1bh\u007f\xf0C\x02\x1e~\r?\xd3h\xfa \xb8\xb1\xbft>\xfeʍ\xa5\x1fJQi&\x9a\x99\xe8\x9b\xe1rW\t\xa6\xeb\xaf\x17\x00&S%\xde\xc0'7E\xc92\xcc/\x00\xc2rh\xcaU@\xf8\xf0\xd6C\xc8\xf6X0\x8f\v\x80*Q\xbe\xbb\xbf\xfb\xfa\xaf\x0f\xbd\xcf\x009\x9aL\xf3\xd2\x12Qѽ#\x90-\v\x1c\xc1\xb8ܢ\xf6L\xdcjU\x10L\x94\xb9\x97>\x12]\xc1Q\x0e\xc9o\xaaM\xc1\xad\xe3\xfb_*4N\xc8\xd5\x1anI\xc5\xc0\x06\xa1*s'\x99k\xb8\x93p\xcb\n\x14\xb7\xcc\xe0\xab3\xc0Qڬ\x1ca\xd3X\xd0Վ\xc3Ξj\x9d\x1fj]6\xc1/\xaf\x10\x1eJ\xccz\x1bƍ\xe2[\x9eѶ\x80\xadҭ\xbe\xf0\xeaj\xdd\x03\x19߲\xaee\x86?HV\x9a\xbd\xb2N\xff\xaa\xca\x0e{\f\x10\xba}\xb8\x1b\f\xa8\x91\t\xa8\x91Z\xa9\f\xe6n\x9f=3n\x1dz#\x98\xe0\x00\xc1W\xd205<\xd24\x95\x01[iI\xbb\xf4\v\xb2\xfc\xf8\xa8\xfed\x10\U0008a135>+\xae`\x83[\xa51\x02W\xa3\x1b\xef:\xa3֎0\x86PR\x95]\xc3\xe3\x1e\x1d\x19Y%l\x90{n\xe0\xedOPpYY\\\x8f\xa0M0\xd8\x13\x85\xc0\xf8\x15\x98G\xf5\xd1xV-\x90\xef\xfdİ\x0e\x11\x9f\xf7h\xf7\xa8\xa1T\xb5\n\x8e\xacr\xcb\x05\x829\x1a\x8bE\xe0x\xad\xf86\x81\xfa$\x14B\x04\x10\x066\xc7\x1a\xe7\xf1:\xddy\xcb6\x02o\xc0\xeaj<\x9d'\xc3F)\x81l\xa8\x84\x87t\xf8\x82\xc6\xf2l\x81\n\x97C2\xf8Q\x11\"\xe8\xf0\x03\xad-B\x87M+k\x96=!\xb0\x9a\x1a\xeep\x10\xa2C\xc4\x1e\x05\xe0\xbf%\xbcw\x9a+s\xfad\x8c-\x04\xcd\xc5Q\x90\xb6\x94\n\x84\x92;\xd4~6w*)K_^\x85\x9c\x1e\xf13\x88\xe9\a\xd2\xf6\x92^m;:tc\xc8\t\xc2\xed\u06dd\xf7\xf0\x1a\xf6p\x03w\xd2\xf9-\x81\x1et#\u09db?\x1f\xfa\xad\xa8\f\x05\x89\xa5\x92+:*ױ\x99<\xb1\x13A*\xdd\xe3\xc8\x18\xb5fR?a\"\xd8Gw\x92\xf8\xf1\xfe\x8eC\xb0\f\xf3:\xc6I\x91yfq\xc73(P\xef\xe6\x0e\x8en+\x9d~OC!Q\xeb\xfav\xa2\x84\xa5\x1d\xedu\v\xaa;\x1a\x82뷕۹\t\xbdjf/v\x9d\b\xc8Ow]^\x11\x1d\xb1d\u007f,R\x97\xe59ݥ2q\u007f\x82\xc6?\x81\x17\xe3\xb3\xdf#\xe6OȂQ\x88\xf4\u007f\xdd1G\x02\xfd\u007fP2\xae\x13\xf6\xf0;\xba\x1a\x15\xd8\x1b\x1b\xa2X\xddi\xdc\f܀\xe3\uf049\xf1UOdq\xca\xe9\x16\x14\xfe Wۑ\xc5r\x05\xcf{e\xfc\x99J\xa1\xd9E\x90\xdc\xc0\xe5\x13\x1e/\xafFz\xe0\xf2N^\xfa\x03\xfedu\xd3X\vJ\x8a#\\\xd2\xd8\xcb\xef1\x82\x12%1\xa9\x1b]A\xa7\x9a\xcaΗ\xac-\x017\xb0\xb9wuf\xee\x1c\xd6IrX*\x13\xb9M\x9a@\xe5^\x19\xeb#\x8b=\xb3\xf4\x94(\x16x\x19\n\xd1+`[\u007f\xf3\xadt}\xa7\xe9\xd4\xde \xe0\xea\xb8f\xe65\xaccc\x13\x11\xf3@\x9dcu\xd9\xee`\xafO/\xfdE'M\xc222.\x16\xe1\x96Zeh̼\x88$h\xeb\x85 a\x13 dށ\xf1\x17\x86\xf3Aɺ\xa5\x1b\xa4\x8eH'\x9a\xf2\x1f\xbeu\xa2\x97n\xf3\xbb\u007f/\tߩx\x01\xed٢`Û\xf1$\x14o\xfd\xc8z\x9b\x04@\xde5л\x8a\xb6z\xba\x05\x19\x04\xe9\xf7pL\x17\\\xde\xd1\x04\xf0\xf6ŏ\xf5FI\xe29\x86\xfbm=\xb6%z\xf3\x81vo\xaaE\xa4(r\xaf\xb1ǹq\x9c\xdb\x19\x8a\x89 \xa5\xb2\xddp\x82\x83[\xaa\xfc\x8d\x81-\xd7\xc6v\x11M\x15\x8aja\xf7\xb7\xedT\xcfI~\xd0\xfa,\xc7\xe9\xb3\x1f\xd9\td\xed\xd5s\x9d_0y\x15\x1bkt)\x84\xc0\xb7\xc0-\xa0\xccT%)\xfc\xe2\xb6:M\xe1Y\xe0\x15t2\xc9\xd2\x14\x84k(\xab\"\x8d\x00+\x92:.g\xe34\xdd\xee\x1f\x19\x17\xaf\xc16;\x95\x86\x11k=\xb6\xd5\xf9\x18\xddD\x91\x82}\xe3EU\x00+\x1c\xe9Sݞ\xad\xcf\xe2\xe8q\xbc\xc9\xe5 \xb8t\x8cX\xe56U)Ц\xeeH\x9f\xb5ᶉ\xe196\as\x90\x02%\x81\xc1\x96q1qy>n'\xd1\xf6\x14_#(\x8b\x97s\"\xd2&_\x11)\x12\x02\xb1\x89\xc6⼶.u\xba\xa9x\xaf1\xcd<[\nJ\xd7\xe6Y\xa9\xb9\x93%\xf5\xd2\x16Z\x101&\x8f?L\xb4Q\xfba\xa2-\xb4\x1f&\xdad\xfba\xa2-\xb7\x1f&Zh?L\xb4\xba\xfd0\xd1~\x98hs\xdd\xe6\xb4\xf5\x12F\xfe\xc5\xc9ď\x8bX$\\Oϡ8\x03?dS\xdc\xfa\xd7'\xa9\x19\x96w\xf1Q\x91\xac\xe0\xf0\xaceE/rb\x12\xd0&]\xb4GI\x93r\xe96H-\xde>\x81~!\t\xf3;\xb2oS\x12~\x96\xd2|\xfay\xa6M\x9aM\x9dh\xaa\xeaI\"t\xa8_\xf68\xb3\xb7\x9bC\xd2\xcf\xd7!;\xb7\xc6\xf4\uf783\x9a\x90\x8a\xb3\x90\x803\x9f\x98;G\xaf\x81\xeb\xd1'\x98\xee%\x8c\xfe~\xe8e\xb1\xf8\\\x06QN{\xc4q\x17\x19\xb2\xf4\x8c#\xb2\x1e\xf2Q\xccQf{\xad\xa4\xaaLpk\x1c\xf4w\xe4]\x85\x1b\x17\xca6\x89\xbd\xbd\x88\xc0|\v{UE\x12Cgh\xb7\x90&4\x9d\x1c\x14\xae\xc2в\xc3\xdbu\xff\x17\xabB\xaa\x10\xecɊi\xde\xffJK&:;\x85\xa8\x9f\"4qF\x9dzg\x96\x9e)\x9d\x9e$4\x9f\xd5sJj\xd00\xf1g\x12\xe8rBP\x8a뼐\xfcsF\xcaOb\xba\xe7w\xdf\f\xa6$\xf5\x9c\x95ʳ\x98\x11\x99\x98\xc0\xd3O͙\ayB\xdaN\x12q\x96StNN\xcc\t\x890\xb3\xebHNlj$\xda\xcc\x02\x9eL\u0099K\xafY\bˍSoғjfAS\xc2\xcdr*\xcd\xcb%̾\x84\x130\xadj\x16\xd3a\x16\x9d\x84y\xfc\x16\x13^NIsY\xa4ؙ)-M\xca\xcaļ\xa7&\xb2\xf4\x13U&\x80\xa6\xa4\xafL\xa4\xa7L@\x9cMZIMJ\x99\x80\xbdp\xec\xceJ\xc9̏\xf1\x97аx\xbe\x89\xbf\x95D\x9d\xbb0\xa5{\xe6⒋\xf2y\xd0\xdd\U00072d9a\xe6\xcdϘ\xe5\xc9\xed\xfet\U000f3a04奠\xfb\x8c\x03ϣ^\xb3\xdd\xe3\xb1y\xd7\xfa\x9b\xa2w^\x9b#A\xfa\xfc\xa5\x11\xcf\xf5\xc0\x88f\x06\x9eQ\b`1\xe1\x1a\xad<\xf3\x8f\xf93\xb5B\xa7\xf3݆\v/vÛ\xff+/\xc1\xf4\x94-\x16\xf2\xb5{,\x1c\x94\xfa\xe9\xef\t\xfe\u05fc\x81\xe8mY\xfa\xf6\x97\n\xf5\x11\xd4\x01uk1,<\xa4\xf0\x1b\xcd8\xef\xa6\xde\xfaA\u007f\xf8\x12\x12\x03ù\xddp\xf0N\xfa#,\nv\x80#\xc1q{^4\xbcv\xea\xcd\xf9\x01\x13]\xe3\x91\x1fՌ\x8e\xfc\xbed{\xa6\xbeBx]\xd7\xe1t\xe7a\xf1\xd8~\x15\a\xe2|\x17b\x06dꫂ\xb4\x1b\xb8\xc5W\x04\xaf\xe5J,9\x13\xc9VT\xda+\x81\xd7x\x1dp«\x80\x13\x9c\x8a\xd3܊d2\xa5d\xff\xbf\x8as\xf1\x8a\xee\xc5k8\x18\xe7\xb9\x18\v \aY\xfd)\xf9\xfaI\xb7\xcb\xc9\x17,)\xb7\xc3\xcbw \xf3y\xf8\t\xf9\xf7\t\xb7#K\x98&\xe4ٟ\x96_\x9f@\xc3Wr>^\xc9\xfdx\r\a\xe4u]\x90E'dQrf\u007f>;\xbc\xaet\x8ez\xf66\"U\xd4f\x85l\xe0/\xf4\xe7\x1c\xc4\xe6\xeb\x127\xaeW\xcf4\x8d\x85\x94\x9b\xe7\xaf\x19\xfc\xc2e\xee\xf9ᄪs\x8e\xf7\xaeHZ\xc3\"\x1e\xa1o\xad\xb6P\x02\xcc߫\x18,\x99\xa6\xbaq\x9b\xa3\xbf\x9b5k\xf8\xc0\xb2\xfd\x00\xfa>\xea'l\x95.\x98\x85\xcb\xe6R\xea\xda\x03w\xff\xbe\\\x03|Tͭ_\xb7\xa4\x84\xe1E)\x8e\xce\x0f\x88\xc0\xbc\xec\x828O \xa2\xc2dB\xbd\xaaP\xc0g\xc1\xf7{\xe8\xf7\x8e\xdcfֵ\x8bj\xb8&\xee\xf80y\x84\xfb\xafd\x9dPŔ\xac\xad\x1e\x13\xec\x8f\xda\xfb\x1b\x16\x97\xf9\xf9\xe5\xef5\x8dU\x9a\xed\xf0W\xe5K\x90-Ѡ\u07fbW\u007f.h\x8d:Ϡ~\x86\x12;MC1\xb4\x01\xb06}hT\b\xcba\x19S'3;\xd1Z\xb1\xb0\x98\xc7\xc7_\xfd\x02,/p\xfd\xbe\xf2\x17\\\xab\x92i\x83\x8e\x9a\xf5\xc2\xfc\xa0\x8d\xfb߽z\x8eE9TX\xf3\xcfC\xbc5R\x8a\x12]U\x9f\x84\xfd\xa1WP\xad&ђ\x88~\x8d\x8f\xea\xb8h\x1d&\xf9\xdd\x1e\x95\xd0)8\x9d\x9a\x92\x14\xbc\xa0'F/[\xf1hJ\u007fOUݣJs\xcbu\xf7|A\xbaPe3$\xbaU\x9a\xca\x15\x85buT\xde\xe7\xac\xd2{ts\xdaޘ6\xf7\xb1杵\xce\xe0\x8c\x19a}\x0fu\x11@\xad߭\xb2L\x80\xac\x8aM4\xa2\xe4\xd6\xd0\f!\xb4for}\x82\xd1\f\v=ѹ\xb4\xb8\x1b\xcd7\xb9\xeaې\xa6t\xf6\xaa\x1b\x00\xe9\xab6U\x96\xa11\xdbJ\x88c\x93'\x95H\x82hhꥉ\xf2\x91qq>E\xfc\xe8\x189&N\x98SY\x1f*c\xa0\xcc\xc3֎\x1e[>\x8f\xf04\x8a\x04f\xf4\xca\x02\xcfS\xe1v<\x82\x8a\xbf꼓\xd0Д\a|f\xa6ex\xcc\xdel\xc1\xf9\x91\xe4\xf09h\x98\x03\x1eP\x82\x92\x94\aGU\xb2|\x81\xe2\xe1\x98\b\xd4.\x94\x90hW\x95B\xb1\xbc>\xfej\xd3.\x14\xb5}\xa4\xc3]\x1fP\xbf130\x9b\x82\x8f\x11\"\x8c\xad-ok\xdd@\xce,\xae\xa2@\x93\f\x83\xa8&\xce\f\xef\x9f\x02\xc9\xda\xec\xf6\xe1nj\xe4\x84\x04\xb7J+ƿQy\xd1\xefT\\㕥j\xac\xf1\xcafU\xd5P1E\x16ת\xaa\x17_&\xed\xd5\xc5Z\x88\xd4\xc9\xdb\a\x940\\\x97\xdb\xf4\x19\xc3\x05\x1a\xc3vu\x11\xc4gg\x9e\xedP\"y\a\xb1\xb0\xbc\x8f\x9f\xb4\x19\xa6\xfd\x12\x80>\xd0\xcb2[\xb10A\x9d \xd3\xe9\xf5&\xe6T\b\xb5\xf3\x85Ry]&\xba\xb6[O\xa4ɷ\x92\xeb\x14;\xf7C\xd3\xd1ц\xeeh\x88\x11mYo\x14|ǝ\x91蘴cz\xc3v\xb8ʔ\x10H\xbav\x8c\xd7kn\u0590\xc7\xfb\x05\x99Y\\\xda\xc7n\xdf\x10\x10\xf4\xdc\xf6\x15t\x98\xcf\x15\xa3*ϖkl˦\x8f\x10R4\xf1Iv\xad\xa7B\xb4\xc0\xf8\x18\xd3n\xdfz\x83\x05\xbd\x1a\xdc\xccPo\xfc*xJ\xf1\xa0O\xc1~S\xfa\n\n.\xdd\u007f\x9cSL\x11\xbbz\xf0I\xf8Sm\xcb\x05\xbc\xef]\x9f\xe69E\xc7\xca\xc4zCL\xf9q\xf1\x14\xfa\x15|±\xdb\xe1\xb3\xe21\xa7\x18u\xac\xaa\xba\xebr'\xef\xb5\xdai4\xe3]\xb5\x82?3n\xb9\xdc}T\xfa^T;.[{\xe3\xa4\xce\xf7L[΄8z|b\x88r\xc9\x04\xff+\x97\xbbw[\x8b)\xb3-\x8cX\x9e\xb2Q̑\xdf\x12\x10\x9e\xfa\xe1=\xbaCy\xd2m\x89\x8bL\xe0\xc0\x92Ԅnm\xf4\x8dK/唫\xbfQ\x95\xed\xa9\xc9V\xcdFD\xbe\x9es\r\x9f\x94\xc5\xfaR\x87\xf7a\xba\x83\x05\x8d]\xe1v\xab\xb4\xf5\xc1\xbe\xd5\n\xf86\xb8A\xb1\xe8\x0e\xe3\x82.\xa5}\x01w\xe0\xb6\xcd\xcbiw&E84)\x18\xaarW\xb0\xa3\xcf\xdbfY\xe6\xbcl\xbc6\x96\x89\x88\xee\xfe\xae4H\xf27\xdd\xce\xc2\xfcO\x11\x1bsD\xf0\xbbn\xff\xa6xFsr\x138O9zW\xe3ϭ\xe8)\x0e\xf4\xda\x02%Jr\xf0\xd5\xd9C\x95A\x12\x83\xb2D\t\xcc\x04|\x12\x1e\x95γu.4i\x99\xb6\xa9\x1e\xd3C\xaf\xf3\x82\xb3D\x90\xe3\xf8>\x84ȯ\u007f\\{;\xfcs.W`\xb8\xac\xff~\x89\x8f+{Q0·\xd2HA\xba\xe8\x1d\xf6\xc8\xfb\xe9\xf9:}\xf4\xff\xb6nΡ9=?\xa4\xd8\xcb_\a\xdd\a/\x14\xa8L\u007f\xd3%ظ\x11z\xfc\x81o\xfd\xb5z\xe6\xb0\xfe\xe3\xdf\xfd\xe5\xc1!\xc9\x1e{3k\x8a\x91\x95\xd5\xd8T\vE\xf9\xef\x05:\x1b\xc9 \xf6\xad\xbc7'\x99\xf3\x87\xf3\x1cԗ\xf4N\xeb?\xad\xf32>\xdb\xe1<\xbf\xf4՜җ]\xdd3\xa3?籴\xc7\xfe\x1c\xbaE\xbc\xd2\x00!\xe2\x97F\x96\xd1x\xaa\x8b~i\xc7-\xadq\x9c\xa8\xf8?pU_\xc81\x8d\x9e\x03\xa3\x8f\xa4@\xf3\xce\xde\x0e3\x85/m$\x9ce\x19:q\xfd4\xfc\x13Z\x97\xfe\xefQ\xd4\u007f%\x8b\xfe\x99)\xe9\x8f[s\x03\xff\xf5?\x17\x10\xaeZ\xbe\xd6\u007f\x0e\xcb}\xfc\xff\x00\x00\x00\xff\xff\xed\xa36\x94nl\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKo#\xb9\x11\xbe\xebW\x14f\x0f\xbe\x8cZ\xb3\xc9!\x81.\x81F\x93\x00\x83x\xd6\xc6\xc8q\x0eI\x80\xa5Ȓ\xc45\x9b\xec\xf0!\xad\x12\xe4\xbf\aŇ\xba\xd5ݲ\xe4A\xb2ˋ->\x8aU_\xbdٓ\xe9t:a\x8d|F\xeb\xa4\xd1s`\x8dğ=j\xfa媗\u07fbJ\x9a\xd9\xfe\xfbɋ\xd4b\x0e\xcb༩\xbf\xa23\xc1r\xfc\x84\x1b\xa9\xa5\x97FOj\xf4L0\xcf\xe6\x13\x00\xa6\xb5\xf1\x8c\xa6\x1d\xfd\x04\xe0F{k\x94B;ݢ\xae^\xc2\x1a\xd7A*\x816\x12/W\xef?T\xbf\xab>L\x00\xb8\xc5x\xfcI\xd6\xe8<\xab\x9b9\xe8\xa0\xd4\x04@\xb3\x1a\xe7\xb0f\xfc%4\xce\x1b˶\xa8\fOwU{ThM%\xcd\xc45\xc8\xe9\xea\xad5\xa1\x99C\xbb\x90(d\xb6\x92H\x1f#\xb1U\"v\x9f\x89\xc5u%\x9d\xff\xf3\xe5=\xf7\xd2\xf9\xb8\xafQ\xc12u\x89\xad\xb8\xc5\xed\x8c\xf5?\xb4WOa\xedTZ\x91z\x1b\x14\xb3\x17\x8eO\x00\x1c7\r\xce!\x9en\x18G1\x01ȘEjS`BD-0\xf5h\xa5\xf6h\x97F\x85Z\x9f\xee\x12踕\x8d\x8f('Y \v\x03E\x1ap\x9e\xf9\xe0\xc0\x05\xbe\x03\xe6`\xb1gR\xb1\xb5\xc2\xd9_4+\xffGz\x00?9\xa3\x1f\x99\xdf͡J\xa7\xaaf\xc7\\YM:z\xec\xcc\xf8#\t༕z;\xc6\xd2=s\xfe\x99))NZ\a\xe9\xc0\xef\x10\x14s\x1e\x8c\xb6e,\x1e?\x97ț\x1c(\xfb[ƪ\x82E\xf6\\\xb3\x81\x0f \xa4\xa3\x02\xc0E\xa2C\xb0\xa8<\xa3\xf59x\x1b\xde$>7z#\xb7C\xa1\xbb5\xcd%\x8b\xb9B\xba\x87\xdc2\xdeD\xa1\x89\xac\xa3\xb1f/\x05\xda)\xf9\x87\xdcH\x9e9\t6e\xae\x8dD%\xdcP\xd2\v^\x16E\xb1(ȫ\x99\xba\xa2\xc3\xe5ic,\x8d\x99\xd4ɂ[\x021\xd8\xd8:\xa7T\xedQ\x8bS5rƍ\x89Qˡ\x80\x83\xf4\xbb\x14\x0e\u0558\xdf\xc1\xab\xbeG\xe3\x05\x8fc\xd3=ޟvH;S\x02Ep\xc8-\xfahm\xa8\xc8|Ȕ*\x80/\xc1ŀڏ\x13e\xc4B\xad\x9c~\xc1\xe3\x10h\xb8\xa6\xdc\\\xc2\\g\xf9\x8eJ\xe7°\xc5\rZ\xd4~4\xa8Sgb5z\x8cq]\x18\xee(\xa4sl\xbc\x9b\x99=ڽ\xc4\xc3\xec`\xec\x8b\xd4\xdb)\x01>\xcd\x1e4\x8bm\xc5\xec\xbb\xf8\xe7\x82\xc8O\x0f\x9f\x1e\xe6\xb0\x10\x02\x8cߡ%\xadm\x82*\x86֩o\xde\xc7\x1c\xfb\x1e\x82\x14\u007f\xb8\xfb\x16\\L\x93<\xe7\x06lV\xd1\xfa\x8fT\xa8E\xa6\b\xa2UҊ\xb1@\x99\x92\x94]gm\xa6X3f\x88c\x15fwP`\xa2\f2\x16Q_p\x18L_q\xb3\\\xec^\xf1\xb1RHK-$\xa7B\xec\xdc7J\x83!\xce\xea\xed\x11\xc1\xfa\x15\xf8\xa5\x880.x\x12 \xe7\xc3+\x1c?t\xf7\xb6mY\nO9\xc79\xf4T@9\xd0H9\x90\xd9!r1(p\xa35y\xa37\xc0N\xa1\xee\xce\xf5c\xfc\x1b#\xc4:\xf0\x17\x1c\x01~ \xcaǸ\xb1`\x9c\x8e\x11/\xc1a\f\xbe\xd7\u0600\xeb6\xce\xd9\x12\xed-\xbc,\x17\xb4\xf1\x94&\x19,\x17\xb0\x0eZ(,\x1c\x1dv\xa8\xa9C\x90\x9b\xe3\xf8]4\x9e\xeeW\x05\xd5Xa\xe4\x1a\xbf`;.C\x8a\xe1sX\x1fGj\x82\x1b\x84l,n\xe4\xcf7\b\xf9\x187\x16\xc0\x1b\xe6w \xb5\x93\x02\x81\x8d\xc0\x9f\x8a\xb5\v\x82\x9e\xf2\xffC\x8e\"ߠ\x9e\u05fc=\xb1\xf3\x16\x87/\x18_\xf1\x9fǼ\xed\x84B\xf9\x9d#\xffy-xɏG%ڟ\x1e\f\xfe\x94*,>\x92*Ϙy\x1e\x9ex\xa5R+\xcf\x16c\xceLu\x81\xb1\x16]c\xb4\xa0\xe6\xe9\xb6:\xade\xf9\u007fW\xad\x8d\xabuz\x1e\xe5zkE\v7\xb5*\xf1\x89\xe6\xcd\xcdJz\xb8\xea\xb6\x02f\xed\xa8Sl\xfb\x95\x9e\x8c\xbfH\x9b\xf2\xaeӧP?\xac!\xe8X\xa9Ō_\xc1\xdf5|\xa2ޖ\xb2\x93\x98\x13\xdfv\xcc\x00\xa4\x03m\x0et\xbcC/\x92\x00\xa3S\xbe\xa6n\x8di\x91\x9b\xe1\xb8t\x90JQƶX\x9b\xfdhƦBӢ:\x02sd:\xfb\xdfT\x1f\xaaw\xbfZ\x17\xa4\x98\xf3\xd4Ԡ\xf8\x8a{9|\xe5\x19\xa2{?8Q\x1c\xff\xe4\x0e\xf4\xe3\xc7\xd2,\xcfl\xde\xf6\xe3\b\x18\x1b\xa9\xa8\x16\x1c\x89\x13m\xc50|\x8f\xfc\xb8\xba\xbfs\xb1\x84G\xed\xc7ʾ\x03Z\x8c\x1d\x13\n\xaa\xe2M~\x97\bΣ\x1d1\x80\x93\xf6\xa2\xceA\x19\xbd\xed9N\x1a\xf9\x95\x82*\xb4dPƂ@O\xa9Io\x81\xef\x98\xdeb\xfb\n\x95\xf9\u007f\x9dS2\x9f\x9eʹ\x16\"\xf5%\xf3\xb8I\xa3Or\xacL\x1f\xbc\x00\xb7\x9b\xc7_\u007f\v\xf7E\xb3\x17ۜ+\xb8\x0f\xf6\x97,M\xa0N}\xfb\"\u070eooo\x87\xcf\xcd7 \xf1ַ\xf0W\xde5\xe0\xc0\\\xfb*\xfe\xeb\xe1PS\xb5z\xb5\x04\xfe\x92v\xa5\xe7\xc3|\x04\xd8\xda\x04\xff\x9agލ\x19t~\xee\u007f\v\x8f\xf1#Ƶ\"\x83\xf6\x14\x8d\xf0`\xa9\x95l_\xc5bP\x18\xcb-\xb7?/-z\xdfZ\xbak\xc3/17\xc85\x9ak\a\x93)_v\xf4\x9aA\xee΄\xf5\xe9\xa5x\x0e\xff\xfeϤMה\x13\x1b\x8f\xe2\x87\xfeǵw)d\x94/d\xf1'\xa7:&}\x1d\x84\xbf\xfdc\x92\xaeB\xf1\\>i\xd1\xe4\u007f\x03\x00\x00\xff\xff\x1d\r\x93\v\x97\x1c\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96Ms\xe36\x0f\xc7\xef\xfa\x14\x98}\x0e{y$\xefN\x0f\xed\xe8\xd6\xcd\xee!\xd36\xe3I2\xb9tz\xa0I\xd8\xe2F\"Y\x00t\xeav\xfa\xdd;$%\xbf\xc8v6=\x947\x91 \xf0\xe7\x0f\x04Ī\xae\xebJ\x05\xfb\x84\xc4ֻ\x16T\xb0\xf8\x87\xa0K_\xdc<\xff\xc0\x8d\xf5\x8b\xed\xc7\xea\xd9:\xd3\xc2Md\xf1\xc3=\xb2\x8f\xa4\xf13\xae\xad\xb3b\xbd\xab\x06\x14e\x94\xa8\xb6\x02P\xceyQi\x9a\xd3'\x80\xf6N\xc8\xf7=R\xbdA\xd7<\xc7\x15\xae\xa2\xed\rRv>\x85\xde~h\xbeo>T\x00\x9a0o\u007f\xb4\x03\xb2\xa8!\xb4\xe0b\xdfW\x00N\r\u0602\xc1\x1e\x05WJ?\xc7@\xf8{D\x16n\xb6\xd8#\xf9\xc6\xfa\x8a\x03\xea\x14xC>\x86\x16\x0e\ve\xff(\xaa\x1c\xe8sv\xf5)\xbb\xba/\xae\xf2joY~\xbaf\xf1\xb3\x1d\xadB\x1fI\xf5\x97\x05e\x03\xb6n\x13{E\x17M*\x00\xd6>`\vwIVP\x1aM\x050\xf2\xc82kP\xc6dª_\x92u\x82t\xe3\xfb8Ldk0Țl\x90L\xf0\xb1\xc3|D\xf0k\x90\x0e\xa1\x84\x03\xf1\xb0\xc2Q\x81\xc9\xfb\x00\xbe\xb2wK%]\vM\xe2\xd5\x14\xd3$d4(\xa8?ͧe\x97\x04\xb3\x90u\x9bk\x12X\x94D\x9eD\xe4\xb8\xd6;\xa0#\xbe\xa7\x02\xb2}\x13:ŧ\xd1\x1f\xf2µ\xc8\xc5f\xfb\xb1\x90\xd6\x1d\x0e\xaa\x1dm}@\xf7\xe3\xf2\xf6黇\x93i8\xd5z!\xb5`\x19Ԥ4\x81+\xd4\xc0;\x04O0x\x9a\xa8r\xb3w\x1a\xc8\a$\xb1\xd3\xd5*㨪\x8efg\x12\xde'\x95\xc5\nL*'\xe4\fm\xbc\x04hƃ\x15\x98\x96\x810\x102\xbaR`'\x8e!\x19)\a~\xf5\x15\xb54\xf0\x80\x94\xdc\x00w>\xf6&U\xe1\x16I\x80P\xfb\x8d\xb3\u007f\xee}s:g\n\xda+9\xe4g\x1a\xf9\xd29\xd5\xc3V\xf5\x11\xff\x0f\xca\x19\x18\xd4\x0e\bS\x14\x88\xee\xc8_6\xe1\x06~I\x98\xac[\xfb\x16:\x91\xc0\xedb\xb1\xb12u\x13\xed\x87!:+\xbbEn\fv\x15\xc5\x13/\fn\xb1_\xb0\xddԊtg\x05\xb5D\u0085\n\xb6\xce\xd2]\xee(\xcd`\xfeGc\xff\xe1\xf7'Z\xcf.H\x19\xb9\xd0_\xc9@*\xf3\x92\xf6\xb2\xb5\x9c\xe2\x00:M%:\xf7_\x1e\x1ea\n\x9d\x931\xa7\x9f\xb9\x1f6\xf2!\x05\t\x98uk\xa4\x92\xc45\xf9!\xfbDg\x82\xb7N\xf2\x87\xee-\xba9~\x8e\xab\xc1\nOW2媁\x9b\xdcbSQ\xc7`\x94\xa0i\xe0\xd6\xc1\x8d\x1a\xb0\xbfQ\x8c\xffy\x02\x12i\xae\x13ط\xa5\xe0\xf8\xef07.Ԏ\x16\xa6\xf6}%_\x17\x8a\xf6!\xa0N\x19L\x10\xd3n\xbb\xb6:\x97\a\xac=\xc1Kgu7\x15\xed\x8c\xee\xbe\xc0\x9b\x93\x85\xcb\x05\x9dơM\xceW\xae\x1e\x1er\xee,\xe1\xec\x16\xd6p\xd6s_璛\xe1\xbf$S:\xf1\xc8FG\"trԟեMoe\x81D\x9e\xcefg\xa2\xbed\xa3\xfc\x04P\xd61(\xb7\x1b7\x82tJ\xe0\x05)\x95\x81\xf61\xf5\x194`\xe2\x19\xbf\x11\xcb\xf1\xbf$\x90\xd7\xc8ܜ\xd9Y\xc1ႦW\xb2\x93Fz^\xa8U\x8f-\bE\xbc\x92YE\xa4v\xb3\xb5\xfc\xcf\xfa\x06\x82e\xb2\xb9\x94\x83\xfd\u007f\xfa\x9bIȸ]\x1c\xce#\xd5p\x87/\x17foݒ\xfc\x86\x90\xe7W>-.\v\xbd\xfdc\xe0\r\x94.^ʳIN\xfd\xce\x1cQd\xf1\xa46\xc7\\9\xae\xf6\xfd\xbb\x85\xbf\xfe\xae\x0e\xf7Zi\x8dA\xd0\xdc\xcd_i\xefޝ<\xb7\xf2\xa7\xf6\xae\xbc\x8c\xb8\x85_\u007f\xabJ(4O\xd3\xeb)M\xfe\x13\x00\x00\xff\xff--\nM\xde\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WO\x93ܦ\x13\xbdϧ\xe8\xf2\xef\xe0_\xaa,\x8d]9$5\xb7d\xed\xc3V\x1cǵ\xe3\xec%\x95\x03\x83z$\xb2\b\b\xdd\xccz\xf3\xe9S\r\xd2\xfc\xd1hfׇp\x134\xcd\xe3\xf1\xfa\x81\x16UU-T0\xf7\x18\xc9x\xb7\x02\x15\f~et\xf2E\xf5ÏT\x1b\xbfܽ[<\x18\u05ec\xe0&\x11\xfb\xfe\x0eɧ\xa8\xf1=n\x8d3l\xbc[\xf4ȪQ\xacV\v\x00\xe5\x9cg%\xdd$\x9f\x00\xda;\x8e\xdeZ\x8cU\x8b\xae~H\x1b\xdc$c\x1b\x8c9\xf9\xb8\xf4\xeem\xfdC\xfdv\x01\xa0#\xe6\xe9_L\x8fĪ\x0f+p\xc9\xda\x05\x80S=\xae\xa0\xf1\x8f\xcez\xd5D\xfc;!1\xd5;\xb4\x18}m\xfc\x82\x02jY\xb4\x8d>\x85\x15\x1c\x06\xca\xdc\x01P\xd9\xcc\xfb!\xcd]I\x93G\xac!\xfeen\xf4\xa3\x19\"\x82MQ\xd9s\x10y\x90\x8ck\x93U\xf1lx\x01@\xda\a\\\xc1'\x81\x11\x94\xc6f\x010\xec=ê\x86\xdd\xedޕT\xba\xc3^\x15\xbc\x00>\xa0\xfb\xe9\xf3\xed\xfd\xf7\xeb\x93n\x80\x06IG\x13838\xc1\f\x86@\xc1\x80\x00\xd8\xefA\x81r\xa0\"\x9b\xad\xd2\f\xdb\xe8{\xd8(\xfd\x90\xc2>+\x80\xdf\xfc\x85\x9a\x81\xd8G\xd5\xe2\x1b\xa0\xa4;P\x92\xaf\x84\x82\xf5-l\x8d\xc5z?)D\x1f0\xb2\x19Y.\xedH\\G\xbd\x13\xe0\xafeo%\n\x1aQ\x15\x12p\x87#?\xd8\ft\x80\xdf\x02w\x86 b\x88H\xe8\x8a\xceN\x12\x83\x04)7젆5FI\x03\xd4\xf9d\x1b\x11\xe3\x0e#CD\xed[g\xfe\xd9\xe7&aH\x16\xb5\x8aG9\x1c\x9aq\x8c\xd1)\v;e\x13\xbe\x01\xe5\x1a\xe8\xd5\x13D\xcc<%w\x94/\x87P\r\xbf\xfa\x88`\xdc֯\xa0c\x0e\xb4Z.[\xc3cQi\xdf\xf7\xc9\x19~Z\xe6\xfa0\x9b\xc4>Ҳ\xc1\x1d\xda%\x99\xb6RQw\x86Qs\x8a\xb8T\xc1T\x19\xba˅U\xf7\xcd\xff\xe2P\x86\xf4\xfa\x04+?\x89̈\xa3q\xed\xd1@\xd6\xfc\x95\x13\x10\xd5\x17\xc1\x94\xa9e\x17\a\xa2\xa5Kع\xfb\xb0\xfe\x02\xe3\xd2\xf90\xa6\xec\x17\xe5\xec'\xd2\xe1\b\x840\xe3\xb6\x18\xcb!f\xe5INtM\xf0\xc6q\xfe\xd0֠\x9b\xd2Oi\xd3\x1b\xa6Q\xccrV5\xdcd\xa7\x81\rB\n\x8dblj\xb8up\xa3z\xb47\x8a\xf0??\x00a\x9a*!\xf6eGpl\x92\xd3\xe0\xc2\xda\xd1\xc0\xe8d\x17\xcekR\xea\xeb\x80ZNO\b\x94\x99fkt.\r\xd8\xfa\b\xeaP\xf9\x03\x81\xf5I\xe6\xf9\xca\xcd\xe0Tl\x91\xa7\xbd\x13,_r\x90,\xffةS\xa3\xf9?\xd6m-^A\x03\x90\xe2\x1e\xdf\xd5g\x19/c\x80Y\xf5\xce\"\x19E,4\b\xafb\x05bRǘΗ\x96\x86.\xf5\xf3\vT\xf0s\xc6\xfcѷW\xc7o\xbcc\x91\xfbՠ{oS\x8fk\xa7\x02u\xfe\x99\xd8[\xc6\xfe\xb7\x80\xb1\\\xa5WC\xc7\x1by\u007fK\x9d\aޡx9^\xde\xc5\x10p\x87\x94\xecEd\x87\xa0\x17\xae\xf7\xa2Mܬo\xbf\x85\x9e\v\xe1W\x0f\xe0BI\x8e-_\xbd\xcf\xebK.\xefQ_2\xa5\xdcG\b\xf2\xa4\x89\x0e\x19\xe9`\x8d\x8f\x86\xbbٌ\x00\x8f\x9d\xd1]\x9e\x98\xc5)\xaeK\xe4\xb5\xc9\x1e\xf6\xed\xf0\xa5\xa6Mę\x02\xa9r\xe1\xcct\v\xf8\xb3\xee\vNti\x81jp\x87\x17\xb9\x19+N\xf4\r~\x96\xe3G\xaau\x8a\x11\x1d\x0fY\xf2\xfd>\x9d\xf0RC\x1b]\xe0\xf7\xbb\x8fϸ\xda\xfbCd~\xc1*\xe3\n\x9a\x10\xb1\"\xd3ʫD\xc6\xc4ײߜ\x93Q\xda\xe9+锨\xd9\x13ů\xc1\x94\x82y\x06\xe2\x87}`1_t\xe5b\x9d\xbe\x03sB\xa4\xfch\xd1j\xfa\\\x92\xb6Ah\xd0\"c\x03\x9b\xa7r\x8b<\x11c\u007f\x8e{\xebc\xafx\x05r\xe1Vlfd$ou\xb5\xb1\xb8\x02\x8e\xe9\x92\xcaf7\x1e:E3ex\xb2\xe7\xcf\x123'\x8c}1^U\x06\\\xf4\xfa\n>\xe1\xe3L\xef\xe7\xe85\x12\xe1y\x19]\xdc\xc9l\x11\x9cu\x92\xbc\x8a\x9a#\x96\x86\xc7\xf6\xd0s(\x19\xa55\x06\xc6\xe6\xd3\xf4\x0f\xe6ի\x93_\x92\xfc\xa9\xbdkL\xf9\xf9\x82?\xfe\\\x94\xac\xd8\u070f\u007f\x1a\xd2\xf9o\x00\x00\x00\xff\xff\xdb\xd9+\xab\xf6\r\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z\xdds\x1b\xb7\x11\u007f\xe7_\xb1\xe3<\xa8\x991\x8f\x89\xdbi;|s\xa4\xa6\xa36\x915\x96\xad\x17\x8f\x1f\xc0\xc3\xf2\x0e\xd1\x1d\x80\x028\xcal&\xff{g\xf1q\xbc\x0f\x90\x944ur/6\xf1\xb1\xf8a\xbfw\xa1\xc5r\xb9\\0-\xee\xd1X\xa1\xe4\x1a\x98\x16\xf8š\xa4_\xb6x\xf8\xbb-\x84Z\xed\xbe_<\b\xc9\xd7p\xd9Y\xa7\xda\xf7hUgJ\xbc\u00ad\x90\xc2\t%\x17-:ƙc\xeb\x05\x00\x93R9FÖ~\x02\x94J:\xa3\x9a\x06ͲBY\xf38\xfa\x01\xed\xe4\xfa\x8b\x99\xdb\x1e\xd1~[aޞ\xc3\xf4\xee\xfb\xe0@\xcb\x1a[\xb6\x8e+\x95F\xf9\xf6\xf6\xfa\xfe\xcfw\xa3a\x00m\x94F\xe3Dr\xe8\xe1\x1bı\xc1(\x8cY}A\x04\xc3*\xe0\x14\xc0\xd0\x06\xab\bc\xc8#\x86 \x0eaIu\rZ\x94nȒ\xf4\xa9-0\tj\xf3\v\x96\xae\x80;4D&\t\xa6Tr\x87Ɓ\xc1RUR\xfc\xb7\xa7mI\xd7\xe8І9\x8cq\xe5\xf0y\xd7/Y\x03;\xd6t\xf8\x1a\x98\xe4в=\x18\xa4S\xa0\x93\x03z~\x89-\xe0ge\x10\x84ܪ5\xd4\xcei\xbb^\xad*\xe1R\xfc.U\xdbvR\xb8\xfdʇb\xb1\xe9\x9c2v\xc5q\x87\xcdʊj\xc9LY\v\x87\xa5\xeb\f\xae\x98\x16K\x0f]\xfa\x18^\xb4\xfc\x1b\x13#\xbe\xbd\x18a\x9d)F\xf8|x=!\x01\n\xb0 ,\xb0\xb85\xdc\xe2\xc0\xe8\xe4 \xdf\xff\xe3\xee\x03\xa4\xa3\xbd0\xa6\xdc\xf7|?l\xb4\a\x11\x10Ä\xdcbt0[\xa3ZO\x13%\xd7JH\xe7\u007f\x94\x8d@9e\xbf\xed6\xadp$\xf7\xffth\x1dɪ\x80K\x9fԐ\xc3\xec4i./\xe0Z\xc2%k\xb1\xb9d\x16\xbf\xba\x00\x88\xd3vI\x8c}\x9a\b\x86\xf9\xd8tq\xe0\xda`\"%MG\xe45Ʉ\xee4\x96$=b \xed\x14[\x11=\x14\xb9s6]^\x8c\b\xe7\r\x97\xbe\xacw\x9a.\x82\\p\x99\xecI\xd8\xe4\xc0\xa7&\x87\x19VΈ\x024S/\xdb\xef\x19F.\x1b\x1dl1\xa3pD\f\xf4I\xc5\xf1\xcc=n\x14\xc7\x1cl\xda\n\xaefA[)\xe3#\u007f\xd4I9?\x85>%\x9f\x05L+~\x06W<\x91\x81\xc1-\x1a\x94%&\xc7u*\x9d\xc9 \x1b&\x1as\x8cǕ\x02Nx\xf5,ⷷ\xd7ɓ'&F\xecn~\xee\x19\xfeз\x15\xd8p\x1f\xe8Ο}q\xbd\r\x87y\x9f\xe6\x140\xd0\x02Cb\xda\a\t\x10\xd2:d\x1c\xd46K\x91\xca' \xc37\x18w\xbc\x0e\x1e,\xba\xcaCh!\xde\x03#\xdf)8\xfc\xeb\xee\xdd\xcd\xea\x9f9\xd6\xf7\xb7\x00V\x96h}^\xee\xb0E\xe9^\xf7\xa5\x02G+\frJ\xfc\xb1h\x99\x14[\xb4\xae\x88g\xa0\xb1\x9f\xde|\xces\x0f\xe0Ge\x00\xbf\xb0V7\xf8\x1aD\xe0x\uf593\xd2\b\x1b\xd8\xd1S\x84G\xe1j1\r\xa6=\aH\xbd\xe2\xb5\x1f\xfdu\x1d{@P\xf1\xba\x1dB#\x1ep\r\xaf|Zs\x80\xf9+\xd9\xceo\xaf\x8eP\xfdS0\xedW\xb4\xe8U\x00\xd7\xc7\xe1\xa1\xd1\x1d@\x06\xcb3\xa2\xaa\xf0\x90UM?\x1fT\xc8U\u007f\v\xca\x10\a\xa4\x1a\x90\xf0\x84Iz\xc1Q\"\x9f\x81\xfe\xf4\xe6\xf3Q\xc4c~\x81\x90\x1c\xbf\xc0\x1b\x10\xb1\xd8Ҋ\u007f[\xc0\a\xaf\x1d{\xe9\xd8\x17:\xa9\xac\x95\xc5c\x9cU\xb2ه\xf6\x881\tʄyp\xcdL\ueffa*\x13C;C\x88\xf6\xcb\xd8\xf6[2\xc9\xe9\xffVXG\xe3/\xe2`'\x9ed\xbe\x1f\xaf\xaf~\x1f\x05\xefċl\xf5H\x02\x1etd\xd8\xe58\x93\x98\xbd\x1f-N\xa9c&c\xed\xd7<+3t\xacʤb\xc3\xf6䩄\xed$\aƭ\x18VY`\x06\x81A\xcb4I\xee\x01\xf7\xcb\x10\xe25\x13\x14\x9f)\x04\xf7}\x0e`Z7\"\x1b\x8ac \x8fIh\xe4\x04\x15ڬ\xb2\xc7\ue795ð\xafsF\n\x1f\aK\x93\f\xcet\x96\\\x9d\xb3\xd4Q\xbfi\x8e\x16e\xd7Ρ,\xe1Ai\xc12\xe3\x06\xad\x13ef\xe2\xd5<\xd38!\xac\xc0\xcb3<\x88-\xe8L\xf1\x12E\x112\xbd\xbe\x80\xf1]\xc7\\\x85p\xbc<8\n\x91*t\xca[\xc7\x10\x97\xf9Rr\xb2\x86J\xabɐV|1ed\xa6\xf3\x98&G\x9d\xd1!\xd2y}\xed\x1b\xdeϨ\xb0C#?\xf24\xf8S\x97\xda\xfbTL\xbc\xb4\xc6.\x15\xe5\xe9㧕\xd3⽜\xef\xf0\xed,ã\xba\x8b\x96\xacw\xd0\xf6\x8fg\xe4\x8ad\x18\x90\v;}\x04#j\xc8}\x12M9\xfe\x96\x89\x069\xa4\xb7\x9d\xe9\x9e\f\xd5!\x95\rn\xc9\xdd\a\xd3K\xa5i\x84\xd7'\xaa5\x82\xf5}\xa2\v{\x82fg\x91\xfb\x9eF\x86\t\xf3\xe4u\xabL\xcb\\\xe8k.\xb3De\xd74l\xd3\xe0\x1a\x9c\xe9\xe6\xd3',\xb1EkYu\xce\x14\u007f\x0e\xabB\xc5\x1e\xb7\x00ۨ\xce\xf5%\xfb\xc8=^بS\xcf\xeb\x1ad\x8b\xe1\xb1:3*VlLڛ\xc6\xef\x19:\x82Ã\xa0G\xb5\xc1|\xd0\u007f\x89O\x00\xf0\x0fZ\xe7\x10Қ\x9c\x81\xf5\xde뤅\xc1\t\xa7|\x83\x8f\x99\xd1\xd9C\xdcp\xf22\x99Lf\xeeGo\rϺ\u007f<\xe8\x1c\v\xe22\xa8U\x93\x8cY9ր\xec\xda\r\x1a\xe2\xc3f\xefЎ\xddy\xae?\xe3\xeb\xba\x03\x1b\a\xfb\x93\xfc\x02\xa5X\xaa\x96L\xfa>*Y\x97S\xc0\x85\xd5\r\xdbg\b\xa7\x8b\xf8܍\x8c\x8b\\\xc0A\x9f\x93Qk4~\xea\xb9}%\x8f\xe9J\xc9#\x95F\xb2g!\xdd_\xffr\"\xd3\x13\xd2a5\t\x0eq\x9e\xd8\xf9\x03\x9d\xf2uN8\x91\xc4Xɴ\xad\x95\xbb\xbe:\xa3\x05w\xfd\xc2d\r\xb3\xe79\xec\xa9EUȉ\xaa\xf7-\xcf2\xd5\xf1\x13\xf09\xa8\xa3\xc5g\xa2P||\xceŠ;\xd4̐\xa5\xfb7\x81\xcb\xe9\xa3\xd5k\xb0\xc27:)\xf3\f\xa9hhCX\nN\x94Z)\x83\x19\x97\t\xf3\xb02\n\"c\xf8\xbfg\xfc\xc8\xea\xc9l\xd0#\xe7\x03ڱY>\x1c\xe96\xfdC\xd0\x1a~\xfdmqHlXI\xc5\x13\xf2\x9b\xe9\x1fYĔ3\xfdՄ\xffY*\x19*\t\xbb\x86O\x9f\x17\xe9\xd9\xf2>\xfd1\x04\r\xfe/\x00\x00\xff\xff\xb0\xddǼ\x99\"\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\x1b\xb9\x11\xbe\xf3Wti\x0f\xcaV\x99\xc3]'\x95\xa4x\xb3\xa5lJɮ\xac2e]\\>\x80\x83\xe6\f\x963\x00\x02`H3[\xfb\xdfS\x8d\a9/\x92\xa2*Z\xcf\xc5\x16\xd0h|\xf8\xd0/4'\xd3\xe9t´xBc\x85\x92s`Z\xe0W\x87\x92\xfe\xb2\xd9\xfa\xef6\x13j\xb6\xf9q\xb2\x16\x92\xcfᦱN\xd5\x1fѪ\xc6\xe4x\x8b+!\x85\x13JNjt\x8c3\xc7\xe6\x13\x00&\xa5r\x8c\x86-\xfd\t\x90+錪*4\xd3\x02e\xb6n\x96\xb8lD\xc5\xd1x\xe5i\xeb\xcd\x0f\xd9߲\x1f&\x00\xb9A\xbf\xfcQ\xd4h\x1d\xab\xf5\x1cdSU\x13\x00\xc9j\x9c\x83V|\xa3\xaa\xa6F\x83\xd6)\x836\xdb`\x85FeBM\xacƜv-\x8cj\xf4\x1c\x0e\x13aqD\x14N\xf3\xa0\xf8\x93\xd7\xf31\xe8\xf1S\x95\xb0\xeeߣ\xd3?\v뼈\xae\x1aê\x11\x1c~\xd6\nY4\x153\xc3\xf9\t\x80͕\xc69\xdc\x13\x14\xcdr\xe4\x13\x80H\x80\x876\x05ƹ\xa7\x94U\x0fFH\x87\xe6\x86T$*\xa7\xc0\xd1\xe6Fh\xe7)\xdb\xeb\x01\xb5\x02W\"m\xe9\xe9fB\nY\xf8\xa1\x00\x01\x9c\x82%BD½2\x80_\xad\x92\x0f̕sȈ\xb8L+\x9eɤ3\xca\x04\xce\xef{\xa3nG\xe7\xb0\xce\bY\x1cC\xf6\u007f\x06\xd5\xc1\xf3\xa0\xf83\x91<\x96\xe8e\x12\x9aFW\x8aq4\xb4y\xc9$\xaf\x10\xc8r\xc1\x19&\xed\n\xcd\x11\x14i\xd9\xe3Nw\x91|J\xfaZ3\x97\xb0s\t\x15A\xb6\xb3\xfdS{\xe8ܾ\x0f\x8a\xc7\x05\x10\x8d\x1a\xacc\xae\xb1`\x9b\xbc\x04f\xe1\x1e\xb7\xb3;\xf9`Ta\xd0\xda\x11\x18^<\xd3%\xb3]\x1c\v?\xf1\xba8V\xca\xd4\xcc\xcdAH\xf7\u05ff\x1c\xc7\x16\x17eN9V\xbd\xdf9\xb4\x1d\xa4\x8f\xfdဖ\x9c\xad\x88\xd7\xffM\xe0.\tҭ\x92]^\xdf\xf7F\xc7\xc0\xb6\x94\xa6@\x9c\r\x82hG뻢\xab\x8f3\x17\x06\xc2\xf4\xe6\xc7\x10\xca\xf2\x12k6\x8f\x92J\xa3|\xf7p\xf7\xf4\xe7Eg\x18@\x1b\xa5\xd18\x91\xa2k\xf8ZY\xa55\n]f\xafIa\x90\x02N\xe9\x04mp\x8a0\x86\xa3f5\xff\xce\xc4\xfck\xaf;X\aN\x17>\x9f\xebN\xdc\x00%;\x10\x16X\\\x1aNq :\x85\xec\x8f\xffX\xdc\xcf\xfe9\xc6\xfc\xfe\x14\xc0\xf2\x1c\xad\xf5\xf9\x1ak\x94\xee\xcd>gs\xb4\xc2 \xa7\xc2\x05\xb3\x9aI\xb1B벸\a\x1a\xfb\xf9\xed\x97q\xf6\x00~R\x06\xf0+\xabu\x85o@\x04\xc6\xf7\xe1/ٌ\xb0\x81\x8e\xbdF\xd8\nW\x8a~\xd2\xda3@\xd6\x15\x8f\xbd\xf5\xc7ul\x8d\xa0\xe2q\x1b\x84J\xacq\x0eW\xbe\x12<\xc0\xfc\x8d\x1c\xeb\xf7\xab#Z\xff\x14\x1c芄\xae\x02\xb8}\xbek{\xe4\x01\xa4+\x99\x03gDQ\xe0\xa1\x10\xed\u007f>xSH\xfc\x1e\x94!\x06\xa4j\xa9\xf0\x8a\xe9\xf6B\x00\xfd\xf9헣\x88\xbb|\x81\x90\x1c\xbf\xc2[\x102p\xa3\x15\xff>\x83Go\x1d;\xe9\xd8W\xda)/\x95\xc5c\xcc*Y\xedB\xb5\xbfA\xb0\xaaF\xd8bUMC\xbd\xc1a\xcbv\xc4B\xba8\xb27\x06\x9a\x19w\xd2ZS\x95\xf1\xf8\xe1\xf6\xc3< #\x83*|\xbc\xa3\xec\xb4\x12T5P\xb9\x10r\x9e\xb7\xc6A\xd2L\x9fm\x82\xf98\x05y\xc9d\x81\xe1\xbc\b\xab\x86\xb2Pv\xfd\x12?\x1e\xa6\xfe\xf4\x8d\x94\x00\xfd\xc0\xf1͒\xe83\x0f\xe7+\xd5g\x1c\xae\xfd\xd6:y\xb8u\xb3D#ѡ?\x1fW\xb9\xa5\xa3娝\x9d\xa9\r\x9a\x8d\xc0\xedl\xab\xccZ\xc8bJ\xa69\r6`g\xfe\xc9<\xfb\xce\xff\xf3\xe2\xb3\xf8\xd7\xf5s\x0f\xd4y\xf4\xbf\xe6\xa9h\x1f;{ѡR\xad\xf8\xfc2q5\xacTN0\x11\f\xe0\f\a\xb1\x994\xf22\x8a\xf6\x13*E?B\xaf\x11oE\xe3!\xf6R\xbb\xa2\x874\x95\xbd]\x84\xd3\xf1\xf7^OF+>\xe9\x93\xd6v\xc9\xde\xe4\xc1\xa1\xfa\x13][\xed\xcdv\x9a\x9c\xed\xd3\f\x9fʾ\x83v\xc9c9t\xed\"\xef!f\xbb\xd4ˣ\aˋ\x9f˹\xa2\xc7@\xf7W\x8b\xd36p3\\\xe1{S\x86G\x9f\x105\xfa7hh8n\x99M\x9b\x8c\xdd7\xb4\xf4\x85\xa5>O\x92:\xe4\xbeT\xa7\x97Ċ\x89\n9\xec\u007f7\xf1\xcdq\xeb\x9b4\xd7c\x95iR\xd4X\xe4>n\x8c\x80\x1e\xaeK}O\xce\x1cNI\xc5@B6UŖ\x15\xce\xc1\x99f8}½j\xb4\x96\x15\xe7\xfc\xeb\x97 \x15^\xf1q\t\xb0\xa5j\xdc\xfe\x19\x1f\x1d-Rqm\xa3\x15\\\xd6J(\x99=\a\xe5\x81d\xc6,n\xef\xf2\xa7M\x0eN\x84\xb2{\u070e\x8c\x0e\xfa\xd0\xedɛdB#s?y븈\x80\xb8\xd19\x0e\xa2\x18\x94\xaaJ֭\x1c%\xa5\xa6^\xa2!\"|\xf3;1\x92\x02\xc7X_Ŀ\xa7\x0eL\x1e4\xa4X\x18T\xc5\x17bΤo\x13\x92\xfd:\x05\\X]\xb1݈\xdet\x12_2\x91\xf9\x92\x1f\x1d,&y!\xb9\xbf\x9f\xbb\xb4\x9f\xb3o\xee\x8f\x17tc?\x15\x8c\xddB\xbb\xefߛ\xdf\xff\xaa\xf1:;\x9c(\xe2\xacc\xc6=7\xec-:\xc2\xe7\"\x9eW=\x1e\xefڡk\x18\xa8\xba\xdb\xfc\x911j\x94\xa8\xc1\xa0G\xce[\xbac/\xb4=\xd2,\xf7\x9d\xfe9\xfc\xf6\xfb\xe4\x90\xeeXNU;\xf2\xfb\xfeOڱVI\xbfP\xfb?s%\xc3O\xcav\x0e\x9f\xbfL 6M\x9f\xd2\xcf\xce4\xf8\xbf\x00\x00\x00\xff\xffe\xe5\xd5&\b \x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc<\xcbr۸\x96{\u007f\xc5)\xcf\"3U\x96ܩY̔w\x19wR\xed\xea\xee\xc4e\xa7ҋ\xa9\xbb\x80\xc8#\tm\x10`\xe3!G\xf7\xd6\xfd\xf7[x\xf1!\x82$$[\xe9\xf4\xe5N\x14p\x80\xf3\xc0y\x13\x17\x8b\xc5\xe2\x82\xd4\xf4\vJE\x05\xbf\x01RS\xfc\xaa\x91\xdb_j\xf9\xf4\xbfjI\xc5\xf5\xee\xed\xc5\x13\xe5\xe5\r\xdc\x1a\xa5E\xf5\x80J\x18Y\xe0\x8f\xb8\xa6\x9cj*\xf8E\x85\x9a\x94D\x93\x9b\v\x00¹\xd0ľV\xf6'@!\xb8\x96\x821\x94\x8b\r\xf2\xe5\x93Y\xe1\xcaPV\xa2t\xc0\xe3һ\x1f\x96\xff\xb3\xfc\xe1\x02\xa0\x90\xe8\xa6\u007f\xa6\x15*M\xaa\xfa\x06\xb8a\xec\x02\x80\x93\no@\xa2\xd2B\xa2Z\ue421\x14K*.T\x8d\x85]l#\x85\xa9o\xa0\xfd\xc3\xcf\t\x1b\xf1H<\xf8\xe9\xee\r\xa3J\xff\xdc}\xfb\vU\xda\xfdS3#\tk\x17s/\x15\xe5\x1bÈl^_\x00\xa8B\xd4x\x03\x1f\xed25)\xb0\xbc\x00\b8\xb9e\x17a\u05fb\xb7\x1eD\xb1Ŋ\xf8\xfd\x00\x88\x1a\xf9\xbb\xfb\xbb/\xff\xfd\xd8{\rP\xa2*$\xad\xb5\xa3L\xd8\x1bP\x05\x04\xbe8\xdc\xec\x06\x1c\x13@o\x89\x06\x89\xb5D\x85\\+\xd0[\x04R\u05cc\x16\x8e\x88\rD\x00\xb1nf)XKQ\xb5\xd0V\xa4x25h\x01\x044\x91\x1b\xd4\xf0\xb3Y\xa1\xe4\xa8QA\xc1\x8c\xd2(\x97\r\xacZ\x8a\x1a\xa5\xa6\x91\xb0\xfe\xe9\xc8Q\xe7\xed\x01.o,\xba~\x14\x94V\x80\xd0o9\x90\f\xcb@!\xbb[\xbd\xa5\xaaE\xed\x10\x9d\x80\x12\xe1 V\xbfc\xa1\x97\xf0\x88҂\x01\xb5\x15\x86\x95V\xeev(-q\n\xb1\xe1\xf4\xef\rle\x11\xb5\x8b2\xa21\xf0\xbb}(\xd7(9a\xb0#\xcc\xe0\x15\x10^BE\xf6 Ѯ\x02\x86w\xe0\xb9!j\t\xbf:\xf6\U00035e01\xadֵ\xba\xb9\xbe\xdeP\x1d\xcfO!\xaa\xcap\xaa\xf7\xd7\xee(Е\xd1B\xaa\xeb\x12wȮ\x15\xdd,\x88,\xb6Tc\xa1\x8d\xc4kRӅ\xdb:wghY\x95\xffѰ\xedMo\xafzo%OiI\xf9\xa6\xf3\x87\x13\xf3\t\x0eX\x81\xf7\xb2\xe4\xa7z,ZB\xdbW\x96:\x0f\xef\x1f?w匪C\xea;\xbaw\x84\xafe\x81%\x18\xe5k\x94\x9e\x89N\xda,L\xe4e-(\xd7\xeeG\xc1(\xf2C\xf2+\xb3\xaa\xa8\xb6|\xffà\xb2\x02-\x96p\xeb\x94\n\xac\x10L]\x12\x8d\xe5\x12\xee8ܒ\n\xd9-Qxv\x06XJ\xab\x85%l\x1e\v\xba\xfa\xf0p\xb0\xa7Z珨\xbcF\xf8\x15N\xffc\x8dE\xef\xc4\xd8it\x1d\x8e9\xac\x85\xec)\a;e\xd9\x03\x9a>\xb4\xf6\xf1\xa7\xdfj\xb0\xc3\u007f\x0e\xb6\xf2\u007f\xcd@+?v\x13\x86\xd3?\f:\x15\xe7O,\x0eT\xca\x00$\xc4\xfd9\xb1X\x0e\xfe\x1f\xa1\xa9}\xf0k\xc1L\x89e\xa3m\a\xb8\x1c\xec\xf8\xfd`\x823G\x84r+\xffV\xfd\xdbm\xf3\xf6_\xabN\x13;&\x12\xc1J \xe5\x1e\x1eP\xee\x90MR\xda>Tc\x95\xd8\xdc$v\xe0\xec\x1cY1\xbc\x01-\r\x8eP\x86HI\xf6#\x84\x89\xb69\x97.\xcd\xf8\xa0\x10\x18-\xb0k(q\f\x12\xfa\x86y\x06.\xb8\xf8\x95J\xac|\\\xfc\xd9Q\xb3}\xe3\x02\x8aw\x1f\u007f\xc4r\x8a<\x90'y\x03D\xde\x1dl\xb6\xbbtp\xf4s\xd1\b\xaeO\x134\xf9\x8c\xc7\x15\x10x½\xf7X\b\a\xcb\x1c\xa2\x9d\xbf\x9b\f\x9f\x86\xc4q\xa9\x17\xef\x1e\xe3ށ\t\xb9\x94\xd9ٹ\xa2\xe0\x9f'L\xf8\xfb\xa9\xa7G@\xbb\xa7\x10\xe1zJ\xda\x17\x8e\x10.\xf2\xce'\x1e\xb8\xbcX\xd4E\xf3\xc8A\xbe\"\x89O\xa4\xfd\th6l\xeb\xe4\x0f\x1dc\xdf(\xcf\"{\n\xb6\xb4\xceDԥ\x0f\x15\xba\xd3\x123c_\b\xa3e\xb3\x90\x97\xfb;>\xee\r\xf7\x9f\x8fB\xdf\xf1+\x1f\x92)'%?\nT\x1f\x85vo\xceBN\xbf\xf1\x13\x88\xe9'\xba\xe3Žڶt\xe8\xa6\xd82\x84\xdb?w>\x93Ұ\x87*\xb8\xe36p\t\xf4p\tS\xbfܴ}\xe8?\x95Q.\x87\xc6\x05_8S\xb9L\xad䉝\tR\xc8\x1eG\x86[k\x16\xf5\vf\x82\xfdl-\x89\x9f\xefS\xc0\x8c\x14X\xc6h\xd3%.\x89\xc6\r-\xa0B\xb9\x992\x1cݧ\xb6\xfa=o\v\x99Z\xd7?GJX\x9ei\x8fOP\xdd\xe5\xfcf\x16\xf6\xe4f\x8c\x8a̞\x1d:\x92\xaf\x1c\x1f:\x8f\x913\xb1\xce\xff\x98\xa5.)KW\\\"\xec\xfe\b\x8d\u007f\x04/\x86\xb6\xdfo\xcc[Ȋ\xd4\xf6\xfc\xfeÚ9'\xd0\xff\x84\x9aP\x99q\x86߹:\x11\xc3\xdeܐ\x19\xeb.cW\xa0\n,\u007fw\x84\r3\xe1\t\xe4\x84\xd5-ȼ!\x17\xeb\x81\xc7r\x05\xcf[\xa1\xbcM]Sd\xa9\x94M\xff\xa1\n.\x9fp\u007fy5\xd0\x03\x97w\xfc\xd2\x1b\xf8\xa3\xd5M\xe3-\b\xce\xf6p\xe9\xe6^\xbe\xc4\tʔĬa<\x99\xe7n\x9f\x9eXts\xddm\x92;\xb8\xb9S\xbbΒ\xc3Z(\xfdS:a7\xb2\x9f\xfb8\xa3\xef\x9b&\xf2^\xb3>{\xc8a5J\xd5zrk\x8d2$\U0007c88d\x11\xc0\vc\xa3\xb9$]\x93\xa0#Mf\xd5\x12xF*|\xcd#g\x8b\xc7x\x8d\x96.G\xfa\xdb\xef\xbfvr\x8c\xf6\x84\xda\xdf]D^۫-DU\x91\xc3*_\xd6Vo\xfd\xcc(\xd3\x01\x90\xe7\xbe\xdc\x18w.\xf3ݽ(C\xae\xbe\xf7L\xf5\x96r \xf1\xf8\xa3\f\x02E\xa0\x16\xf3\x9a\xc8?[\xa2`\x85ț\xdc\xf8\xf7`\xaf+\xca\xef\xdc\x02\xf0\xf6\xd5\xed;\xb4\xe4:\x89\x9d\x91\xd4\rC\x9b\x17\xce\xe2\xe4\xbaF\xa2\x84\xe7-J\xecI\xc50\xe1m=\xc6L\x90\\\xe8n^\xc1\u00adE\xf9F\xc1\x9aJ\xa5\xbb\x1b\xcd\x158\xa3r\xc5\xe1H\x0e[\xec>\xd3\n\x85\xd1'\xf0\xe0};\xbbW\xa0\xad\xc8WZ\x99\nH%L\x86q\xf7\x8f\xb5/\xb4j\xaa\xa8\x81\x03τꦞ\xe42,ZX.\xd5\fu.\x8bW\xb8\xb6\xea\xa8\x10\\\xd1\x12e\xac\xf2{\xceRa\x0f\xee\x9aPfR\xe5\x9b\xd4sl\x98\xca\xdfKyR\x94\xfa\xc9\xcf\xecd\r\xb7\xe2\xb9O\xa0l\x12l\xc9\x0e\x81\xae\x81j@^X\xbe\xa0\xf4*\xdb-\x11\x88\xe1H\x93-\x96y\n\xde>\xc8M\x95G\x80\x85;ٔO&ź\xc3?\x10\xca\xce\xc16+y\xa7\x1f\x8d\xdf\xda\xd9\xdf\xe4h4J%߄\xad\x10\x1e\x90\x94\xfbx>\x88\xd66Tu2 @\x1a\xdeՈg8\x19\xc7\xc4wa\x17\xaf\x19\xb8QN3\x18{\x90ϧ\xba\xeb\xedX\x10g\xf5v\xec\x02\x8d\xa1;%5s\xd7\x03`Met\x9c\xdd\xde\x1b\xa99\xc2\xf3Y\xa1\rP\xb1\xf4I/k>\x83\x1f\xed{\x97F\xca\xe0I\xec\x8ew]\xb28\u06dd\x90\x9f%\xfa\xbah\xdb\x15\x16.)(w\xb80\xfc\x89\x8bg\xbep1\xa5\x9a\xcd\xd67\x8b\x9f\xac8\xbe\xa5\xd2\xe8\x8bW\xbe\bD\xfb{\x06\xa5\x90\xcd\xe6\xa3\x02\xe3))\x98SC\xbe\x8du\xe4\xcf\xd9]L\xad?19\xd4\x1co}\xffino\xd3]zV\xc7\u007fxޢޢ\x8c\x8d\xad\v\xd7ÛR\xabmi\xb2u\x85\x9bf'+?ћ\xf2Mx\a\xedOi_\x99\x1bƮ\xac`\x13ôoE\x95&!DY=@+!\x18\x92ö\u061c\"\xfa\\\xe9\xbc\xdf\x0f֔\xaecC\x98\x88\x8b$0\xf4\xbc\xf4]\x9fݺl\xbf\x06\xee\xb2?q\xa7\u007fz\xabXFy{\xa6\xa8=\xdd@7E\xaf\xa1\xd8t)\xd6\xca`\x18\x17:+\xbf+\xf2\xcd\x14\xa2\xc7\xcb\xcf!ي\x9a\xec\xde.\xfb\xffh\x11\x8a\xd1.\xb3\x90@\xe5y\xdb\xe4\t\x9c\xe5\xe5%\xdd\xd1\xd2\x10֓\xc0\x0e\xcdZ҂\x90\xc0)Kա,\xcd\xe3\xfc\x1e\x8d\xe1S\xed\xf3\xd1G\x9f\xd5iw'\xaff}r\xa5\xba_\x89\x1e\xd1\xe0Ǧf\xf3[\xf2\xf2k\xd1\xd3\xc5\xe3c*Ї\xf5\xe5Q\xa0\xf3u\xe7\x1cOu\xa6\xc6|Be9\xb3\xab\xe8\xc5\t\xe8\x9c\xda\xf1I\x15\xe3\xd9ƛ\xcc:q\xbf\x02<\r\xf2\x88\xeap\x16q\xe6+\xc1G\xd7\u007fC\xbdu\x12\x8f\xec\xaao\xa2\x9e;\tx\xb4\xd6;Uŝ&y\xa2\u009b_\xbb\x9d\x04\xed\xea\xba\xf3\x15\xdb\xd7\xeb\xcbz\r\x17y\\\xd5\xccV]_\xe4Bg\xd4U\x8f\xa9\xa6\xceR\xec\xc4\xcaiS\x19\x1dY\xf7\xd8zi\xbf\x1e:\x024\xa7J:R\x05\x1d\x818Y\x1bͭ}\x8e\xc0\x9e1\xbb\x93R2\xf1g\xe3u\xffJ\xea\x9a\xf2͐\xf3\xb9\xf21)\x1b\x83\xd2iw͞pt\x9d\xe3^X\x91Z\xd2\u007f\x90\x98\bAb҉r-\x96\xf0\x8e\xef\ap]3t\xd2\xe5\xee\u007f\xb1b\xb7\xf5L\x19\xeb~\x95\xe1\xc0vA\x85\x0f\x9cT:\x10\xb6\x03\xc7>fJ2EȞ\xbf;\x17q|:\x18\xdeMcM\xfb\xcf)י\xea\xed\x89\xfese\x98\xa6u\xf2\x10\xd7R\xec\xa8K\x8amq\xdf\xd0\xf3wᾇX\xed\x1d\xa4O\x0f\xcd\xf9Z\x1e\x84\x02$u*\x9e\x911 j\x88~\xe1\xbf\t,\xc4\xc2}\xe6c9\x19\xe5!|;x\xe5\xce`*@\xe5\xf1k\xb5ʂq\xdf\x15\xaaD\x02`ԺL{\xb8\xde\x19w\xef\xfe0(\xf7 v\xae\n\x1a\\\x9e\x99\x86c\xaf)\x94am\x87GP\x80\xfeK\xd4\x03Ͽ\xd5\x18\xf0\x8e{\x1b\x9c\x04{\xb0G\a\xc7*\xad6ڱ\xfa\xd9\x062#C\x93P\xb9hf\xa7\xe5a\xd2\xd4\xe4v\xeb\x9e7\xf69>\xfa\x99\xf5;\xce\x12\x01\x9d\x1e\x03M\x80\xcc\xed\xbe\xcd\xcb\xd8\xcfv۞+\x16\x9a\x8b\x86\xb2\xdd\xc0\xbcn\xdast\xd1\x1e\xd1={DTt\\\\\x94M\xa6\x9c.ٳDGg\x8c\x8f\xce\x11!\x9d\x16#̀<\xe8~\xcd\xe9k\xcd*2e\x97(r\x8aJ\xf3u\xcd\xe9~Ռ>Ռ\xe2\xc7\xdcN3\xfaQ\x8f\xebC͠ᙢ\xa73\xc5O爠\xce\x1bC\xcdFQ\xb3\x923\xf9\xf7\xc99\xf2XM\xfd(J\xbc\x17R\xcf9\xfc\xf7\x87\xe3\x13\x15\xacN\x10$X\t<\x0eM \xe5|\xf9\xe0ǟ\x86T\xba\xd8\x14ֿ\xff2\x87\xcfC3p\x1a\x11\xeb\x92\xc6\xf8,\x81\x87\x9d\xefpQ\x9c\xd4j\x9b\b\xef^\x8ẹ&\xdad\xe2\xe3\xc7\xf6P\xa2ŶS\xb6y\xc6X=\x94\x9d\xab\x87\x0e\xf6d\x1d\x1e\x0fȝ'g\v9eW\x1d\x0f\xfb\xdbT#2\xbfl>\xf9\x9bfO\x9e\x91s\xe9BF\x1b\xd5EYh\xe9rB\x1dbV\x17g|~8m@2?\x8a=\xf9s\xd8yb%\b5\xf6%l\xce\u05ee\u007f*='Ԯ*\xb6X\x1a\x86\x19\x97\xd4\xcdB\xa6o`\x92r>U\xe9\xfau\x83?\xeb\x9al\x1d{\x19\v]\xb6\x15*E6\xf1Z\xa1g\x94\b\x1b\xe4\x96\xc4Sw̴\x8d\xca\xe1\x047\xfd\x12\x96Z\xa4І\x84\x05\xbc\xa5l\x92\xb8\xa9\x04\xa0\xbf\xb9\xcc\x0e!\x9b\xd1sC\xb9\xc6\xcd }\x1a\x9a\xa4\x1f\x90\xa8Û\xee\x06\x84\xf8\xd0\x1d\x1b\x82_O\x03\xffU5q<\xf5\x17\xa3i*q\xcaC\x10n壮˪\xb7Dͩ\xcb{;\xa6\xf9z\xa0s(\x1bM\xf90\xb2\xa7t7\xf3\x02>\xe2s\xe2\xed\a'\xf4.\xa1\x91>J\v\xb8\xe3\xf7Rll\x8c\x91\xf8\xf37B5\xe5\x9b\x0fB\xde3\xb3\xa1\xfcS<\x93\xc7\r\xbe'RS\xc2\xd8\xde\xef'1\xf76\x1e\xe6\xc4\u007f\xf3\xb3G\xfe\x98bR\xc0y6\x1a\xf0\xc3\xda\xe0\x88r\u007f\xd0]\xef\xfeJ\x18\xdd=\x15oT{`ҙY\am\t\x1f\x85Ƙt\xa3}\xa0T\xc1\n\x95^\xe0z-\xa4\xf6\xc1\xd8b\x01t\x1d\x14u*\xc8 \x949_\xc3\xdf\xd3g\x1d\x90\xa6\xf0\xdbX>!\x81\xf0=Hw*\x9c\x93R\x91\xbdo\xad#Ea\xac\x1e\xb8V\x9a\xa4\fڋ\\[\xe7\xdc\x04i\x1e\xc9K\xf4=\xb5\xee\xf8\xe6+@S\xadP\xba\xdea\xfb\xb7'\x9d\xfb\xac\xc0\xab\xa0d\xc1\x01\xdc\xc7\a\x9d\xaf\x9a@\xd9㜎\x8f\xa7\x94\x8f\xfb_h\xc2\xee\xc6\x1d\xb5~#m38\"\xe0\xa6\x0f\xd1\xe8]H6\xde&DU\x9cjyVl\t\xdfX\xf1\x91\xc2l\xb6Q\x04\xc74\xf5X>ĸ\xfb\xd0jwRUL^k#y'\xf7\x12\xd2\xd9e\xbb\xdd)\xa0\xd3$\x9c\xf23{\xa6u\xce\xd3\xec\r~\x99\v\xe1\x16\xb6\x0e\xc4\xf7k\xfaw\x8d\xee~\x9f\xe3\x04|9\x18~\xd0 i݁\x16b0\xdc\t\xe2\xfc']\xc7\xfblW\f\xffk0\xe2\x1b7:>\x13\xc9)\xdf\xcc!\xff[\x18\x96\xf0\x81\x02\x84\x84\x17\x94@\xa2\xf1\x8b\x8e\xf2\x82\xe2&G\xaell<\xa3\x17\xf8A\xc934x\xe9\x04\xb9\xec\x109\xac\x14\u07b4\xf1\x03)\n\xacuh@\xee\xde\xd6|y\xe9~\xc4\xeb\x98\xdd\xcfBp\xaf\x16\xd4\r\xfc\xff\xdf.\"B_\xe2\xad\xcb\xf6\xe5\xbf\x02\x00\x00\xff\xff鐱=\xdaZ\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c7l\xef\xfa\x15\x18\xf5\xc1\xed\x8c\xee\x14O\x1f\xdaћ++SM\\\xdbc)zi\xfb\xc0\xdb\xc5\xe9\x18\xed\x92\x1b\x92{\xf25\x93\xff\xde\x01\xc8\xfd\xbc\xfd\xe0\x9e\xa5i\x9a\x11_\x12\xed\x91 \b\x80\xf8 A\xf8l\xb5Z\x9d\x89B>\xa0\xb1R\xab+\x10\x85\xc4\xef\x0e\x15\xfde\xd7O\xffj\xd7R_\xeeߟ=I\x95^\xc1ui\x9dο\xa1եI\xf0#n\xa5\x92Nju\x96\xa3\x13\xa9p\xe2\xea\f@(\xa5\x9d\xa0ϖ\xfe\x04H\xb4rFg\x19\x9a\xd5#\xaa\xf5S\xb9\xc1M)\xb3\x14\r\x03\xaf\xa6\xde\xff\xb4\xfe\x97\xf5Og\x00\x89A\x1e~/s\xb4N\xe4\xc5\x15\xa82\xcb\xce\x00\x94\xc8\xf1\nl\xb2ô\xccЮ\xf7\x98\xa1\xd1k\xa9\xcfl\x81\t\xcd\xf6htY\\A\xf3\x83\x1f\x140\xf1\xab\xb8\v\xe3\xf9S&\xad\xfb\xa5\xf3\xf9\x93\xb4\x8e\u007f*\xb2҈\xac5\x1f\u007f\xb5R=\x96\x990\xcd\xf73\x00\x9b\xe8\x02\xaf\xe03MU\x88\x04\xd33\x80\xb00\x9ez\x05\"M\x99T\"\xfbj\xa4rh\xaeuV\xe6\x15\x89V\x90\xa2M\x8c,\x1c\x93\xe2\xce\tWZ\xd0[p;l\xcfC\xed7\xab\xd5W\xe1vW\xb0\xb6\xdco]섭~\xf5$\xf2\x00\xc2'w ܬ3R=\x0e\xcd\xf6\x01\xae\x8dV\x80\xdf\v\x83\x96P\x86\x949\xab\x1e\xe1y\x87\n\x9c\x06S*F\xe5\xdfD\xf2T\x16\x03\x88\x14\x98\xac{x\x06L\xba\x1f\xe7p\xb9\xdf!d\xc2:p2G\x10aBx\x16\x96q\xd8j\x03n'\xedh\x1ao`%2؋\xac\xc4\v\x10*\x85\\\x1c\xc0 \xcd\x02\xa5j\xc1\xe3.v\r\xff\xa1\r\x82T[}\x05;\xe7\n{uy\xf9(]\xa5b\x13\x9d祒\xeep\xc9\xdaRnJ\xa7\x8d\xbdLq\x8f٥\x95\x8f+a\x92\x9dt\x98\xb8\xd2\xe0\xa5(\xe4\x8aQW\xacf\xd7y\xfa\x0f\x15G\xed\xbb\x0e\xaeG\xfb\xcd7V\x84\x13\x1c \x8d\xe8\x05\xc6\x0f\xf5\xabh\bM\x9f\x88:\xdfn\xee\xee\xdb\xc2$m\x9f\xfaL\xf7\x96\x845, \x82I\xb5Ű\xa3\xb7F\xe7\f\x13UZh\xa9\x1c\xff\x91d\x12U\x9f\xfc\xb6\xdc\xe4\xd2\x11\xdf\u007f/\xd1:\xe2\xd5\x1a\xae\xd9\xee\x90\x1c\x96\x05\xed\xc0t\r\xb7\n\xaeE\x8eٵ\xb0\xf8\xea\f J\xdb\x15\x116\x8e\x05m\x93\xd9\xef\xec\xa9\xd6\xfa\xa12o#\xfc\xaa\xf6\xf8]\x81Ig\xcb\xd08\xb9\x95\to\f֞\xb5\n\xe8iP߆w-\xff\xc2j\xaa\xff\xb5\x87\x87\xd7eլh\xc9~\xb8\x1ds\xb81c$W\x1e\x1a\xe9\x14\xa5\xfb\xdc\x1d҂-J\x04(3\x98t\xb5^\xac};\x82\tAխGp<\xe2*\xff\x84yAjc\x06\xc5\xfbЍP$\xfa\xa4\xb5;U\x19\xfeJ\xcd\xea\xa0]\xe1H\xb9\xf1t;$\xbe\xede\x1a\xb4\xd7\x11Wa\x92\xb3\xd4\x12+\xef\x94(\xecN;\xb2q\xbatC\xbdz\v\xb8\xbe\xbb\xed\rjq\x9e\xb0b\x1bΌv\x1a\x9e\x85<\xe6\xb4o$\x97\xd7w\xb7\xf0@.\x11V0\xc1[rp\xa5Q\xac\x8e\xbf\xa1H\x0f\xf7\xfaW\x8b\x90\x96\xac\x95*\xbb|1\x02x\x83[\xda\xf4\x06\t\x06\r@ch\x0fXFM\x97n\xcd\x0eG\x8a[Qf.(9i\xe1\xfdO\x90KU:<\xe6;L\xf3\xde\x13\x89\xc1\xf9\xd5\xd8{\xfd\xb3\xf5\x8c\x8c \xe9Ǒ\xa1\x03[\xaa\xd0)\xec\xb9\xdf\x18Ue\x86`\x0f\xd6a\x0e\x9b\x00\xa5\xb6\xd5\xcc\x15\xd6\aY\x16\xc0X\xd8\x1c*܇\xd7M^\xb8\xd8dx\x05Δ\xc3\xd3Nm\xdd!\xda|C\xebd\x12A\x99\xf3>i\xfc\xc8\x01\xc2\x18\xfea\x84(=\n\x90\x91\x17O\xe4h\x06\n\x91\xb7\x90e-\xe2\xceS\x05\xe0\xbf\x14|$\x03\x97\x90ٹ\n\xe6Lb\xc6&TiȴzD\xe3g$W\xe1Yf\x19oi\xcc\xf5\xbe\xe3d\xb5\x1b\xd9\x16\x83\x19\x19Iؖdv\xd6@\xb2?*#RY\x87\"]\x9f\xbf\x16\xf3\xf0{\x92\x95)\xa6u\x983\xa8Kz\x8c\xbb9\x1a\xc4\x01\xa1\x90\x8a43\x85_DtU\xff:B\r\xf65\x85A2\x18 \x95\x87I\xa4!E\xb3\x19Q\xd2Ԥ\xc3|\x04\xcfٝ\xbc\x80j\xc2\x18q\x98\xa0Y\x154/!Y=&\xb8b\x99L\x90\x88U;\\L5&\xcd\xc8\xfa\xfe\x1f\x12l\xa7\xf5S\f\x91\xfe\x9d\xfa5\x8e%$|6\x01\x1b܉\xbd\xd4\xc6\xf6\xa3\x13\xfc\x8eI\xe9Fw\x9bp\x90\xca\xed\x16\r\xc1›\x8e\xbf\xa7\x885mV\xa9\x99i\xc6\x1f\xad\xaba:1\x8f\xa91\xb6\x14v_F\xa1\x02#NV\x8fuC*\xf72-E\xc6jB\xa8įO\xd4\xf8\x8d)\xb7\x19\x818\xc2\xdf+\xa3j\x15ĥ\x8eW\xaa\x15\x92ۗk3f\xb7|;\x063N\x86\x8d`gr̅k\x9a)3\xb4\x01\x15o\xfe\x1a\xbds\xd1p\xca\at\x99\xd8`\x06\x163L\x9c6\xe3\xe4\x89\x11\x02\xdfb\xf5\xe7\be\a4i\xd7ߚU\xa2M#\x87l'\x93\x9d7V$e\f\vR\x8d\x965\x86(\x8a\xec0\xb5h\x88\x91\x8c0ٜ\xd2hZ\x84\xfa\xe8\xc3\x1dS$M\x8b\xd4\xc1M\x9b\xd1\xc6]\xaa\xd7b\xf3F\xf4\x0e\x9aꇄ\xfd\xf6h\xf8\xcb\v;\x91[\xa2]\xc3\xed\x160/\xdc\xe1\x02\xa4\xab\xbe\xc6@%W\xb1\xc1\xe3oƸ\xd3v\xcbm\u007f\xf4\x8b\xef\x96\x17\xe1Z\x8d\xc6߄il\xac\ue0adZİO\xed\x91\x17 \xb75\xc3\xd2\v\x8a!\x1d\xb2/5\x87h\xcbљ\xe5\xdcK\x12(\xd6\xf6R˅Kv7\xf51PĈ\x1e\xad\xfa\x00\xbc_^\xc50̃\b\x90P;\x15|\x82)\r\xe6\xfed\xf4\x9e\xf7G\xf3\x85=\xc0\x0f\x9f?b:G2\x88\x97ԣE}\xe8y:m\x14x\x81Q [\x8bb7\xad\x8e\xf1\xfc\xf9\xf7\x05\bx\u0083\xf7\xac\x06\x83ˡF\xac\x155H\x83|\x18\xcfj\xe4\t\x0f\f*\x9c\xaeG\xc1[\"*\xbe=\xe1!\xb6k\x8f\xa8\x84_8\xd7\xf3ԥ\x0f\xbc\x8a\x98\xadԴ\x9a\xa8a\xef\x80\xd3q\x8b\x85eJ\xa9j\x15\xc5O\\vͰΕ\xd2\x13\x1e\xdeY\xcf>\xda5;Y,\xa0\x00)l\xb0\xc8;\xac\xbaKy\x10\x99L\xeb\xc9x\x9f,\x80x\xab.\xe0\xb3v\xf4\x9f\x9b\xef\xd2\x12\x8a*\x85\x8f\x1a\xedg\xed\xf8˫\x92\xd8/\xe2D\x02\xfb\xc1\xbc-\x957\vD\x97E\xf378\xb0\t%\x11\xad\xd9&-\xdc*\x8a\xcf<}\x96\xb0i\x87\x15r\x1e\xad\xbc\xb4|\x1b\xa3\xb4Z\xb1\x99\xaef[\x00\xb4\x8dW`\x956\x1dN],\x848\x88b@\uf7ac\x95\xff\xe5\xe8\x1ek\xaa\x19,2\x91`Z\x9dJ\xf3\xa5\x99p\xf8(\x13\xc8\xd1<\"\x14d7\xe2\x85j\x81&\xf7\xed\x04)\x8cw-\xaa\x16\xcc\xc2\xc0\x1d\xd0P[Ѯ\x8f\xecY\xb19\xaa\xfb\xc8\r\xd9t\xf7\xb8U\xb2yg\u007f(\x8a\xfa픎e\x96e!\xbf\x8e}\x10\x8f\xa4w?r\xc1\xc7\xd6\u007f\x90ye\xf1\xfe3\xce\x1a\ni\xec\x1a>pBK\x86\xed\xf1\xd5)ak\xaa(\x90\x84\x89\xb4@r\xb2\x17\x19\xb9\x0f\xa4\xbc\x15`\xe6\x9d\t\xbd=\xf2\xa0\xe2T\xcc\xf3N[o\xf3\xebc\xf5\xf3'<\x9c_\x1ci\xaf\xf3[u\x1e\a\x93t\xfe\x91Ҫ\xbd\x16\xad\xb2\x03\x9c\xf3o\xe7\xec\x98-\xd9\"'8o\v\xa4:\xba+\xa7\x8e,\t\x05(֮\xbc\x16\x1a\\'X\x90\v?\xb7\x8ah\x99.\xb4\x1d\xb9]\x1cA뫶\xce\x1f\x00v\xdc\xed\x81\x13\u0098\xe8/\x9c\x1a\x82\xd8:4`\x9d6U2\x03\xa9\xdd\xde\x019q\xde\xce\xf3\x9eX]\x9fFz\xc0\x14d\x9e7\x1a\xc2\xeb\xf4s\x9f\xe5@\xff?\x0f3ag\x89a\x17F'h\xed\xbc(EZ\x8e\x99\x03\xdb\xfa\xb0V\xf8\xe0m\x1b\xa5\x9ac\x8e\x92\xab\xb6\xcc\x15'Ҟ\x10\xd8\xdc|o\x9d;\x93\x1a\xa2\xbfcD\xf9\x14\x1c\x81\x13\x1d\xf3\\\xf4\x13k\xa2ѽ\xf6\xa3\xab\r\x18\x80\xf9\x80\xc9<\x96\xacT\x96\xf9\xcdA$\xffj\x8eG.\xd5-O\x04\xef_\xcdY\x81J\x95㩡\xccu5\xbeaH\xfd!6~\x85*=C\xf3]\x8d\xc1\x0eg\x8fo2\xe29\x05\xe4L+\xedڇ5a\xa6w\x16\xb6\xd2X\xd7 \xbc\x00\xaa\xb4|M\xfd\xba1\xa6\xba1\xe6\xe4\x10\xf3\x8b\x1f\xdd:V\xdc\xe9\xe7\x90Դ$\xb0\xae\x88\xbf\x13{\x04\xb9\x05\xe9\x00U\xa2K\xc5\a^\xa4.h\x9a\x05\x10=\x13\xbd1\x89\xb4\x99\xad\xc1\xaa\xcc\xe3\t\xb2b\xe9\x94j\xf6t\xac=\xe4g!\xe3N\xa7\xe04\xb6\xba\xa9ġ\xa1\xd6͆\n\x19D\xed\xec\xb5\\|\x97y\x99\x83ȉ-K\xe2ƭ\xcf=\xaaR\xdd<\xaf\x9f\x85t!\x83\xd8_\xac.Ӧ\x89\u038b\f\x1dVYE\x89VV\xa6X\xbb\x0f\x81\xff\x839ZcM\xc0VȬ4\vt\xf4b\xce,\x8dۂzz\xf9`,\x1e\x91\x15\x133\xf2\xd0}\x81\xd3\x1f\xa8\xc5\xe7\xa5-\xcdF\xab\xf3\x87\xe6\xad\xca\x0f\xe4\xa0-{ 0\x9bo\x16\x834\xc4d\x99\r\xe7\x8f\xcd@]\x92[\x16\x1b\x83G\xe4\x91\xc5g\x8fő\a\xf8\xb5}l\xceX\xb4\xd7\x16\x9b\x1f\xf6:Ya\x91\xb9`\xad\f\xafY\x90'f\x80E\x13,.\xdb+:ǫ\x95\xb95O\xad\x89̮\xe1|\xadY\x90C\xf9\\1YZQ\xb8F\xe7f\xd5\x19W\xf3'\x89?\x94\x91\xf5\xf2\xb9\xdf/\xe9\xe7O\xe7WEeUE\xc5\x02\xf38G\xe5M-͖\x8a\xa2\xea\xd2̨:\xebib\xe2\xa8|\xa8\xe3\\\xa7\xa9\xa5\xccfA\x8dg8M\x81\x1d\xca}\x8a\xc8k\x9a\x00\xd9\xcexZ\xec\x06\xccJ\xd3L\x87\xe1\x8a\x18U\x9b\xb7\xb5\xd9\xff\x85\x04\xfe袵\xe9\xb8\xc01Qɗ\xde\x10\xe2}\xe5\xf5\r\xb9\xd5\xe31\x9ew\xb6Op\xabG@\xden!/3'\x8b\xacU\x92\xc2\xed\xf0P?y\xffM\xf3\xd3\xcb́\xa1}\xf9V\v\xf0\x18\xc8n\x80 ,\xe8\a]?\xfc\xcc%\xfa\x1a\u007f\xd1]R\xec\x15|\xdc5\xcf\xfc\xf3\x95\xc8g+\x91\x97@1\xd8G>OY\xfe,%\x92\xce'\x06[\x93SG>?Y\x14n\x9d\x18pMB\x9czn2\x1drM\x1f\xa7\xf5\x9f\x99\x9c\xe0NDH\xd8l\x97\x1f\xbe\x11\xd0&E3{\xb9\xb2D4g\x85\xb2\x17\x13u\xe7\xefU\x1d\xa8\xaa{Q\xaf\xf6\xc5\xcd\x18wt\xfd\n>\x81_\xa4J=oH\b[\xfeE\xe7\xf6\xa7q~\xc6\xee\x80\xda\x1eg(\x8b鯍,\x16\x82\x14)\x87E|\xbdm\xd7p#\x92]=\xc3\bH\x9ew',l\xb5Ʌ\x83\xf3\xfa>\xee\xd2O@\u007f\x9f\xaf\x01~\xd6\xf5Eh\xab\xca\xcd\bT+\xf3\";P\xf4\x03\xe7m0?&8\xa3\xc2gC\xb9\xbfP\xcf,\"\xfe\xbd\xeb\x8e\x18*7\x19ʺU\xb0'\xd8,\xd4\x01\xbe>\xb0\x17\xc5ş\x92\xa6HV\xf0\x91\xaa\b\xb8WCk\x04\xe4X\xbd\xc8E\xc4\x1a\xbf\x14\xb6N\x1b\U000487f4/\xe9\x19C\xad\xee\x88NUנ\xa3\xaa\x14\x91\xf0\xe6kdeU1\xe6>\xc0&s\xec\xa8\xca a;\xa6\xbcf\xf6\xb7sY\xc4\xe2\xee\xef?\xf9\x059\x99\xe3\xfac\xe9/\xe8W\x850\x16\x89\xd2\xd5B\xfd\xa0\u0378}\xdb\xe9g.\xd3\u05ee\xbb٪l\x8c\x9c\xa9ƹ\x00'\xadfߩlY\x91.F\xd8\x1f\x86G\xb6\xd4I\x8b\x89SW\xfaz;\nKX\xab\x13\xc9\x1a\x88\x8f\x828A\xec\xf5J\xc4MY\x92\teQZ\xfc\xf2\xac\xd0|\xab6\xaa\xbdUc\x855;$\xfc\xf5h\xe0hQM\xa7Y\xef\xf5\xba\x0f\xd9;\x15\bd}\x11\xd2\xeaLKں\xf4\xec1\xe9f\xf6\xff\xf8\xde\x1f\xf6YW\xc3\xd5^Wu\x01ڳ\b\xca\xfa\"\xab1\xb5\x85}5\xd6D\x14\xae4\xc1\xac&\xa5\xe1zy\x04\x04}9\xb9Ӫ\v7\xd5\xdagx\xd9\xd4oo\xa2\xfc\xd9j\xf1\x03\xfc\xab\xeb\x03\x8f\x16\xcc\xf56\xd5Ws_\x11\xfc\xd3\xd89\xb8\x0f\xb8\xbe\xe0\\-e\xeaS'\xf7\x06B\xf3\xc0\xaa.\xe1\xdd\x18\xea\xc3ٚ+\xf8\x8c\xcf\x03_o\x14-\xe2\xf8.ͧdb\xcag\x03C\x95\xd5'\x97\xb8\xafGq>쀶誹^\xf7^\xa2\rW\xa9\xad\xbb\xf8\xdc\xd7!\xb6\xfe\xa3\xdc\xfa\x83\x9b\x84\xd6\xf4OG=F\x15פ\xd2\x1aSX\x83[\xea\xe8\xa3E\xb3粰\x95\x90\x04\x1b\xde\xfeRn\x9a2\x91\xf0ǟgͮ\x14I\x82\x85\v\t]\xed\u007f\xc5\xe2ܗy\xad\xfe\x91\n\xfe3\xd1\xca;\xd8\xf6\n\xfe\xf3\xbf\xcf \x18\xe0\x87\xea_\xa2\xa0\x8f\xff\x1b\x00\x00\xff\xff\t\xb7x\x1e\xf3c\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Msܸrw\xfd\x8a.\xe5\xb0I\x95f\xbc\xae\x1c\x92\xd2͑\xbd\x15\xd5sl\x97\xa5\xe7K\x92\x03\x86\xec\xd1`E\x02|\x008\xf2\xe4\xd5\xfb\xef\xa9n\x00\xfc\x1ar\b\x8e\xa4\xca\xe6\x95p\xd9\x15\ah4\xba\x1b\xfd\x014\xda\x17\xab\xd5\xeaBT\xf2\a\x1a+\xb5\xba\x06QI\xfc\xe9P\xd1_v\xfd\xf8\xafv-\xf5\xbb\xfd\xfb\x8bG\xa9\xf2k\xb8\xa9\xad\xd3\xe5w\xb4\xba6\x19~ĭT\xd2I\xad.Jt\"\x17N\\_\x00\b\xa5\xb4\x13\xf4\xd9ҟ\x00\x99V\xce\xe8\xa2@\xb3z@\xb5~\xac7\xb8\xa9e\x91\xa3a\xe0q\xea\xfd\xaf\xeb\u007fY\xffz\x01\x90\x19\xe4\xe1\xf7\xb2D\xebDY]\x83\xaa\x8b\xe2\x02@\x89\x12\xaf\xc1f;\xcc\xeb\x02\xedz\x8f\x05\x1a\xbd\x96\xfa\xc2V\x98\xd1l\x0fF\xd7\xd55\xb4?\xf8A\x01\x13\xbf\x8a\xbb0\x9e?\x15Һ?\xf5>\u007f\x96\xd6\xf1OUQ\x1bQt\xe6\xe3\xafV\xaa\x87\xba\x10\xa6\xfd~\x01`3]\xe15|\xa1\xa9*\x91a~\x01\x10\x16\xc6S\xaf@\xe49\x93J\x14ߌT\x0e͍.\xea2\x92h\x059\xda\xcc\xc8\xca1)\xee\x9cp\xb5\x05\xbd\x05\xb7\xc3\xee<\xd4~\xb7Z}\x13nw\rk\xcb\xfd\xd6\xd5N\xd8\xf8\xab'\x91\a\x10>\xb9\x03\xe1f\x9d\x91\xeaal\xb6\x0fpc\xb4\x02\xfcY\x19\xb4\x842\xe4\xccY\xf5\x00O;T\xe04\x98Z1*\xff&\xb2Ǻ\x1aA\xa4\xc2l=\xc03`\xd2\xff8\x87\xcb\xfd\x0e\xa1\x10ց\x93%\x82\b\x13\u0093\xb0\x8c\xc3V\x1bp;i\xe7iB@z\xd8zt>\x0f?{\x84r\xe10\xa0\xd3\x01\x15\xa5z}$\x91=\x98\x1f\x1e0\x01\x18\x93\xa8\x12\xb5e\xe1hG\u007f\xeb~\xf2\x006Z\x17(\xd4E\xdbi\xff\xde\xcb^\xb6\xc3R\\\x87κB\xf5\xe1\xdb\xed\x8f\u007f\xbe\xeb}\x86\x81,\x05J\x81\xb4 \xe0\ao\f0a\v\x83\xdb\t\a\x06\x89\xf3\xa8\x1c\xf5\xa8\f\xae\"u\xf3\x06$\x806P\xa1\x91:\x97Y\xe4\n\x0f\xb6;]\x179l\x90\x18\xb4n\x06TFWh\x9c\x8c[Ϸ\x8e\xaa\xe9|\x1d`\xfc\v-\xca\xf7\U00092216\x85/l(\xcc\x03\x1d\xfc\xfe\x90\xb6ş\x99\xd4\x03\f\xd4I(Л\xdf1sk\xb8CC`\"֙V{4D\x81L?(\xf9?\rlKR\xefX\x18\x1d\x06}\xd06\xde\xc0J\x14\xb0\x17E\x8dW T\x0e\xa58\x80A\x9a\x05jՁ\xc7]\xec\x1a\xfeC\x1b\x04\xa9\xb6\xfa\x1av\xceU\xf6\xfaݻ\a颊\xcdtY\xd6J\xba\xc3;֖rS;m\xec\xbb\x1c\xf7X\xbc\xb3\xf2a%L\xb6\x93\x0e3W\x1b|'*\xb9b\xd4\x15\xab\xd9u\x99\xffC\xe4\xa8\xfd\xa5\x87\xeb\xd1~\xf3\x8d\x15\xe1\t\x0e\x90F\xf4\x02\xe3\x87\xfaU\xb4\x84\xa6OD\x9d\xef\x9f\xee\xee\xbb\xc2$\xed\x90\xfaL\xf7\x8e\x84\xb5, \x82I\xb5Ű\xa3\xb7F\x97\f\x13U^i\xa9\x1c\xff\x91\x15\x12Ր\xfc\xb6ޔ\xd2\x11\xdf\xffR\xa3uī5ܰ\xdd!9\xac+ځ\xf9\x1an\x15܈\x12\x8b\x1ba\xf1\xd5\x19@\x94\xb6+\"l\x1a\v\xba&s\xd8\xd9S\xad\xf3C4o\x13\xfc\x8a{\xfc\xae¬\xb7eh\x9c\xdcʌ7\x06k\xcfF\x05\f4\xa8o㻖\u007fa55\xfc:\xc0\xc3\xeb\xb28+Z\xb2\x1fn\xc7\x1cn\xcd\x18ɕ\x87F:E\xe9!wǴ`\x87\x12\x01\xca\f&}\xad\x97jߎ`BPu\xeb\t\x1c\x8f\xb8\xca?aY\x91ژA\xf1>t#\x14\x89>y\xe3NE\xc3\x1fլ\x0e\xda\x15\x8e\x94\x1bO\xb7C\xe2\xdb^\xe6A{\x1dq\x15Nr\x96Zf\xe5\x9d\x12\x95\xddiG6N\xd7n\xac\xd7`\x017w\xb7\x83A\x1d\xce\x13VlÙ\xd1NÓ\x90ǜ\xf6\x8d\xe4\xf2\xe6\xee\x16~\x90K\x84\x11&xK\x0e\xae6\x8a\xd5\xf1w\x14\xf9\xe1^\xff\xd9\"\xe45k\xa5h\x97\xaf&\x00opK\x9b\xde \xc1\xa0\x01h\f\xed\x01˨\xe9ڭ\xd9\xe1\xc8q+\xea\xc2\x05%'-\xbc\xff\x15J\xa9j\x87\xc7|\x87Ӽ\xf7Dbp~5\xf6^\xfff=#\x13H\xfaqb\xe8Ȗ\xaat\x0e{\xee7EUY \u0603uX\xc2&@il5s\x85\xf5AQ\x040\x166\x87\x88\xfb\xf8\xba\xc9\v\x17\x9b\x02\xaf\xc1\x99z|\xdaS[w\x8c6\xdf\xd1:\x99%P\xe6rH\x1a?r\x840\x86\u007f\x98 ʀ\x02d\xe4\xc5#9\x9a\x81B\xe4-\x14E\x87\xb8\xf3T\x01\xf8/\x05\x1f\xc9\xc0edv\xae\x839\x93X\xb0\tU\x1a\n\xad\x1e\xd0\xf8\x19\xc9Ux\x92E\xc1[\x1aK\xbd\xef9Y\xddF\xb6\xc5`AF\x12\xb65\x99\x9d5\x90\xecOʈT֡\xc8ח\xaf\xc5<\xfc\x99\x15u\x8ey\x13\xe6\x8c\xea\x92\x01\xe3>\x1d\r\xe2\x80PHE\x9a\x99\xc2/\"\xbaj~\x9d\xa0\x06\xfb\x9a\xc2 \x19\f\x90\xca\xc3$Ґ\xa2\xd9L(ij\xd2a9\x81\xe7\xecN^@5a\x8c8\x9c\xa0Y\f\x9a\x97\x90\xac\x19\x13\\\xb1BfH\xc4j\x1c.\xa6\x1a\x93fb}\xff\x0f\t\xb6\xd3\xfa1\x85H\xffN\xfdZ\xc7\x122>\x9b\x80\r\xee\xc4^jc\x87\xd1\t\xfeĬv\x93\xbbM8\xc8\xe5v\x8b\x86`q@\xdd\xc4ߧ\x88uڬR3\xa7\x19\u007f\xb4\xae\x96\xe9\xc4<\xa6\xc6\xd4R\xd8}\x99\x84\n\x8c8Y=\xd6\r\xb9\xdc˼\x16\x05\xab\t\xa12\xbf>\xd1\xe07\xa5\xdcf\x04\xe2\b\u007f\xaf\x8c\xe2*\x88K=\xafT+$\xb7\xaf\xd4f\xcan\xf9v\ff\x9a\f\x1b\xc1\xce\xe4\x94\v\xd76S\x17h\x03*\xde\xfc\xb5z\xe7\xaa\xe5\x94\x0f\xe8\n\xb1\xc1\x02,\x16\x989m\xa6ɓ\"\x04\xbe\xa5\xea\xcf\tʎhҾ\xbf5\xabD\xdbF\x0e\xd9Nf;o\xacH\xca\x18\x16\xe4\x1a-k\fQU\xc5\xe1Ԣ!E2\xc2dsJ\xa3m\t\xeac\bwJ\x91\xb4-Q\a\xb7mF\x1b\xf7\xa9ވ\xcd\x1b\xd1{h\xaag\t\xfb\xed\xd1\xf0\x97\x17v\"\xb7D\xbb\x86\xdb-`Y\xb9\xc3\x15H\x17\xbf\xa6@%W\xb1\xc5\xe3\xef\x8cq\xe7\xed\x96\xdb\xe1\xe8\x17\xdf-/µ\x06\x8d\xbf\x13\xa6\xb1\xb1\xba\v\xb6j\x11\xc3>wG^\x81\xdc6\f˯(\x86tȾ\xd4\x1c\xa2\x1dGg\x96s/I\xa0T\xdbK\xad\x14.\xdb}j\x8e\x81\x12F\fh5\x04\xe0\xfd\xf2\x18\xc30\x0f\x12@B\xe3T\xf0\t\xa64X\xfa\x93\xd1{\xde\x1f\xed\x17\xf6\x00?|\xf9\x88\xf9\x1c\xc9 ]R\x8f\x16\xf5a\xe0\xe9tQ\xe0\x05&\x81\xec,\x8aݴ&\xc6\xf3\xe7\xdfW \xe0\x11\x0f\u07b3\x1a\r.\xc7\x1a\xb1V4 \r\xf2a<\xab\x91G<0\xa8p\xba\x9e\x04o\x89\xa8\xf8\xf6\x88\x87Ԯ\x03\xa2\x12~\xe1\\\xcfS\x97>\xf0*R\xb6R\xdb\x1a\xa2\x86\xbd\x03N\xa7-\x16\x96)\xa5\xd8\"\xc5\xcf\\vðޕ\xd2#\x1e~\xb1\x9e}\xb4kv\xb2Z@\x01R\xd8`\x91wX\xbcK\xf9!\n\x997\x93\xf1>Y\x00\xf1V]\xc1\x17\xed\xe8?\x9f~JK(\xaa\x1c>j\xb4_\xb4\xe3/\xafJb\xbf\x883\t\xec\a\xf3\xb6T\xde,\x10]\x16\xcd\xdf\xe2\xc0&\x94D\xb4a\x9b\xb4p\xab(>\xf3\xf4Y¦\x1dF\xe4qCv\xba{\xda*ټ\xb3?\x94D\xfdnJ\xc72˲\x90_\xc7>\x88Gһ\x1f\xa5\xe0c뿒ye\xf1\xfe[\x9a5\x14\xd2\xd85|\xe0\x84\x96\x02\xbb\xe3\xe3)ag\xaa$\x90\x84\x89\xb4@r\xb2\x17\x05\xb9\x0f\xa4\xbc\x15`\xe1\x9d\t\xbd=\xf2\xa0\xd2T\xcc\xd3N[o\xf3\x9bc\xf5\xcbG<\\^\x1di\xaf\xcb[u\x99\x06\x93t\xfe\x91\xd2j\xbc\x16\xad\x8a\x03\\\xf2o\x97\xec\x98-\xd9\"g8o\v\xa4:\xb9+\xa7\x8e,\t\x05(֎^\v\rn\x12,ȅ\x9f[E\xb2LW\xdaN\xdc.N\xa0\xf5M[\xe7\x0f\x00{\xee\xf6\xc8\taJ\xf4\x17N\rAl\x1d\x1a\xb0N\x9b\x98\xcc@jwp@N\x9c\xb7\xf3\xbc'V7\xa7\x91\x1e0\x05\x99\x97\xad\x86\xf0:\xfd\xd2g9\xd0\xff\xcf\xc3\xcc\xd8Ybؕ\xd1\x19Z;/J\x89\x96c\xe6\xc0\xb69\xac\x15>x\xdb&\xa9攣\xe4ؖ\xb9\xe2D\xda3\x02\x9bO?;\xe7Τ\x86\xe8\xef\x14Q>\aG\xe0DDz\x14\xc3Ědto\xfc\xe8\xb8\x01\x030\x1f0\x99\x87\x9a\x95\xca2\xbf9\x88\xe4\x1f\xcd\xf1(\xa5\xba\xe5\x89\xe0\xfd\xab9+\x10U9\x9e\x1b\xca\xdc\xc4\xf1-C\x9a\x0f\xa9\xf1+\xc4\xf4\f\xcdw5\x06{\x9c=\xbe\xc9H\xe7\x14\x903\xad\xb4\xeb\x1eք\x99~\xb1\xb0\x95ƺ\x16\xe1\x05P\xa5\xe5k\xea\u05cd1\xd5'c\xce\x0e1\xbf\xfaѝcŝ~\nIMK\x02\xebH\xfc\x9d\xd8#\xc8-H\a\xa82]+>\xf0\"uA\xd3,\x80\xe8\x99\xe8\x8dI\xa2\xcd\xec\fVu\x99N\x90\x15K\xa7T\xb3\xa7c\xdd!\xbf\t\x99v:\x05\xe7\xb1՝J\x1c\x1ak\xfdl\xa8\x90A\xd4\xcd^+\xc5OY\xd6%\x88\x92ز$n\xdc\xfaܣ\x98\xea\xe6y\xfd$\xa4\v\x19\xc4\xfebu\x996\xcdtY\x15\xe80f\x15eZY\x99c\xe3>\x04\xfe\x8f\xe6hM5\x01[!\x8b\xda,\xd0ы9\xb34n\v\xea\xe9僱tDVL\xcc\xc4C\xf7\x05N\xf3\xbc\xfd\xa8\xcc2\x97\xf9\x9b\xc1\x97wM+#IJ\xf5\x9cw:\v\x93\xbd\u05few\x1a\x84W\xa8Ô{:\v\x951ysO\x9b\xf6枾\xb9\xa7o\xee頽\xb9\xa7o\xee\xe9\x9b{:\xde\xde\xdc\xd3N{sO\x93\xedG\n\x86+>\xb9=\xd1!\t\xab\xc4\x14\x8c9\xb4g\xe6\n\x99F7Em\x1d\x9a%\x19ҷ\xe3#G^\x03d\xbeˊ_\xe8NIM\x9b\xba\xd2\x1a\xbd&e\x9a\xb6d\xdcL\xfe\x1dU\x82\x17\xfe\x02\xd9\xf6\xa9\ttsis\xfd\xdc\xf1&]\xcd\xff\xdf\x04A\x9c\x8e\xd3\a\xee\xf9\xf7yݜ\xab~\xee\x1b\xc7\x01\x11\xe3?d^ybZ\xdbL2\xdb\xe9D\xfc)\v\x1fi9\xb8\\\xe8\x13\xd3\xf4\x12\xbf\xffشtX~\xad\xc2vH\u007f0v;2\xec\x19OƄ=\xa8lg\xb4ҵ\ra\"\xcd\xf0!\xf3o\xe8\xe2D\xf6\xf8\x8d\xd7\x14\x9b,\xbc\x87\x9d\xae'\x92\xbdg蚐\x827\x9dx\x17\xaerщ\xfd\xfbu\xff\x17\xa7C\x1a\xde\x04\xd6O\xd2\xed\xfcCF\x8a\xe7\xd5C7\xd7?n\xde\xf0\x98y(x\x13\x10\xb5\x01%\v/\x95\x11BO&\xe1k\xe5\xcf\r\xceV~\xf3\xd1kz\xb2\xde\xd2\x14\xbd&\xa9j\xde\xd4>#1o٫\x89\xd9$\xbc\x14\xa4!%\xf5n<\xa9n\x06ꒄ\xbbԃ\x89\x84\xe4\xba\xf4\x94\xba4\xf2\x00\x97 HM\xa4KveS\x93\xe6^'U.1A\xae\x93\xf66\v\xf2̴\xb8d\x82\xa5\xa5\xc0%'\xbeu\xd2\xd9\xe6\xa9u\"\xddm<\x89m\x16\xe4X\x92[J\xeaZ\x12\xae\xc9\tkM\x1a\xda\xfc\xf1\xea\xb3\xd2\xd4^>!\xfe%\x83\x9f\xd3IgI\xa9fI\x01\xd2<\xceI\xc9dKSȒ\xa8\xba4]\xacI\x05;1qR\x92\xd8q\x02ة\xa5̦\x86M\xa7}\x9d\x02;\x96\x10\x96\x90\xecu\x02d7\rl\xb1\x1b0+M3\x1d\xc6˄\xc46ok\x8b\xff\v\t|\ue8b5\xe9\xb9\xc0)\xa1\xda\xd7\xc1\x10\xe2}\xf4\xfa\xc6\xdc\xea\xe9\xc0\xd7;\xdbg\xb8\xd5\x13 o\xb7Pօ\x93Uѩ\xd3\xe1vxh\xea\x00\xfc\xae\xf9=\xea\xe6\xc0о~o\x04x\nd?@\x10\x16\x9e\xb0(\xe8\xbfGT\xc8|U\x9cL\xaf\x90l\xce\xf4\xdd@\xa8\u007f\x10J\xea\\\xf9\x04H~\xac\xcb\xf6\xac$H\xb1l\xc2\x191\xe9ig\xd7\xfb\xe8\xfc\xed/5\x9a\x03\xe8=\x9aƫ\x99\x14\xb3\xf6\x11Wؚ\x96\"\xbc\xa8J\x82N\xf2\xb5\x99\xfa\xaaez74\x1b\x1a>(of\x87\xb82,\xd2!mptJuR,4\x05B\xe9\x06\xc2\xc4\xf8\x14_zɫ\xa6\xd7\b\x95^\"XJr+^#`z\xad\x90iiд\xe4>7\xe9U\xd2k\x84NK\x82\xa7E\x1e`\xfa\xab\xa3\xd7zm\xf4\nA\xd4\xd9a\xd4\"ҥ\xbe&Z\x1cL%\xaco\xe6\xf5БǕ\x00r\xf2\xd5\xd0x@\x95\x00\xf1\xe8\xb5\xd0lH\x95\xb2\x0f\x86A׳\xdf\xfe$\xe76,\xba`K\xcdKH\xbb\xfb\x9a\u007fӓ\xf8\x96'\xf1f,\x05\xfb\xc47;\xcb\xdf\xea$\xd2\xf9\xcc`\xeb\xe4ԉor\x16\x85[g\x06\\'!\x9ez\x83s:\xe4:}\x9c6|{s\x86;\x91 a\xb3]\x9e}M\xa2M\x8ef\xf6\xc6i\x89h\xce\n\xe5 &\xea\xcf?\xb8k\x89%ϨW\xf76k\x8a;\xba)\r\x90\xc1\x9f\xa4\xca=oH\b;\xfeE\xefJ\xacu~\xa6o\\Z\x8f3\xd4\n\xf5wi\x16+A\x8a\x94\xc3\"\xbe\xf3\xb7k\xf8$\xb2]3\xc3\x04H\x9ew',l\xb5)\x85\x83\xcb\xe6\x92\U0009d7c0\xfe\xbe\\\x03\xfc\xa6\x9b\xdb\xe1N\xe9\x9f\t\xa8V\x96Uq\xa0\xe8\a.\xbb`\x9e'8\x93\xc2gC\r\xc4P\xe4-!\xfe\xbd\xeb\x8f\x18\xab\xc1\x19j\xddE\xd8'\xd8,\xd4\x01\xbe\xfd`/\x8a+bem\xe5\xb0\xe0#\xc5\bxPXl\x02\xe4T\x11\xcdEĚ\xbe)\xb7N\x1b\U000407f5\xafs\x9aB\xad\xfe\x88^\xa9۠\xa3b\xdeLx\b7\xb1\xb2X\xa1z\b\xb0M\xa7;*\xbdH\xd8N)\xaf\x99\xfd\xed\\\x91\xb0\xb8\xfb\xfb\xcf~AN\x96\xb8\xfeX\xfbk\xd0U%\x8cE\xa2t\\\xa8\x1f\xb4\x99\xb6o;\xfdĵ\v\xbb\xc5H;垑\xd3\xf78A\xe2\xac\xd5\xec{\xe5>#\xe9R\x84\xfd\xc7\xf8Ȏ:\xe90\xf1T\x9e\x83\xdeN\xc2\x12\xd6\xeaL\xb2\x06\xe2\xa3 Κ{\xbd\xbay\xa7,\xc9\teQ[\xfc\xfa\xa4\xd0|\x8f\x1b\xd5ު\xa9j\xa3=\x12\xfe\xf9h\xe0d\xa5Q\xa7Y\xef\r\xba\x8f\xd9;\x15\bd}e\xd6x\xa6%mS\x8f\xf7\x98t3\xfb\u007fz\xef\x8f\xfb\xac\xab\xf1\x12\xb8\xab\xa6*\xefE\x02e}\xe5ٔ\x82˾Dm&*W\x9b`V\xb3\xdap\x11A\x02\x82\xbe\xc6\xdey%\x97\xdb\x12\xf63\xbcl\x8bڷQ\xfel\t\xfd\x11\xfe5E\x93'\xab\b{\x9b\xeaKܯ\b\xfey\xec\x1c\xdd\a\\tq\xae\xc04\xf5i2\x9e\x03\xa1y`,\xd6x7\x85\xfax\n\xeb\n\xbe\xe0\xd3\xc8\xd7O\x8a\x16q|\x97\xe6\xf3T1糁\xb1r\xf3'\x97\xb8oFq\x92\xf0\x88\xb6諹A\xf7A\xf6\x11\x97\xeem\xba\xf8\x84\xe01\xb6\xfe\xa3\xdc\xfa\x83\x9b\x8c\xd6\xf4OG=&\x15\xd7I\xa55\xa5\xb0F\xb7\xd4\xd1G\x8bfϵr\xa3\x90\x04\x1b\xde\xfdRo\xdaڙ\xf0\u05ff]\xb4\xbbRd\x19V.d\xb9u\xffi\x8fK_\xfb6\xfe\xcb\x1d\xfcg\xa6\x95w\xb0\xed5\xfc\xe7\u007f_@0\xc0?\xe2?\xcfA\x1f\xff7\x00\x00\xff\xff\xfa\xbf\x85\x18\be\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VO\x8f\xeb4\x10\xbf\xe7S\x8c\x1e\x87w!\xe9{\xe2\x00\xca\r\x15\x0e+`\xb5\xda>\xed\x05qp\x9di;\xacc\x9b\xf1\xb8K\xf9\xf4\xc8v\xb2m\x93\x94]\x90\xf0-\xf6\xfc\xf9\xcdo\xfed\xaa\xba\xae+\xe5\xe9\t9\x90\xb3-(O\xf8\xa7\xa0M_\xa1y\xfe.4\xe4V\xc7\xcf\xd53ٮ\x85u\f\xe2\xfaG\f.\xb2\xc6\x1fpG\x96\x84\x9c\xadz\x14\xd5)Qm\x05\xa0\xacu\xa2\xd2uH\x9f\x00\xdaYag\fr\xbdG\xdb<\xc7-n#\x99\x0e9\x1b\x1f]\x1f?5\xdf6\x9f*\x00͘տP\x8fAT\xef[\xb0ј\n\xc0\xaa\x1e[\b\xc8II\x94\xc4\xc0\xf8G\xc4 \xa19\xa2Av\r\xb9*x\xd4\xc9\xf1\x9e]\xf4-\x9c\x1f\x8a\xfe\x00\xaa\x04\xb4ɦ6\xd9\xd4c1\x95_\r\x05\xf9\xe9\x96\xc4\xcf4Hy\x13Y\x99e@Y \x1c\x1c\xcb\xfd\xd9i\r!py!\xbb\x8fF\xf1\xa2r\x05\x10\xb4\xf3\xd8B\xd6\xf5JcW\x01\fLe[\xf5\xc0\xc5\xf1s1\xa7\x0fث\xe2\x04\xc0y\xb4\xdf?\xdc=}\xb3\xb9\xba\x06\xe80h&/\x99\xef\x85Ȁ\x02(\x18P\x808PZc\b\xa0#3Z\x81\x82\x12\xc8\xee\x1c\xf79G\xaf\xa6\x01\xd4\xd6E\x019 d\x05\xd9*\x03Ge\"~\r\xcavЫ\x130&/\x10텽,\x12\x1a\xf8\xc51f2[8\x88\xf8ЮV{\x92\xb1\xeb\xb4\xeb\xfbhIN\xab\xdc@\xb4\x8d\xe28\xac:<\xa2Y\x05\xda\u05ca\xf5\x81\x04\xb5Dƕ\xf2Tg\xe86w^\xd3w_\xf1Ч\xe1\xe3\x15V9\xa5\xca\n\xc2d\xf7\x17\x0f\xb9!\xfe!\x03\xa9\x1dJ}\x14\xd5\x12ř\xe8t\x95\xd8y\xfcq\xf3\x05F\xd79\x19S\xf63\xefg\xc5pNA\"\x8c\xec\x0e\xb9$qǮ\xcf6\xd1vޑ-ե\r\xa1\x9d\xd2\x1f\xe2\xb6'\tc\xed\xa6\\5\xb0Σ\b\xb6\b\xd1wJ\xb0k\xe0\xce\xc2Z\xf5h\xd6*\xe0\xff\x9e\x80\xc4t\xa8\x13\xb1\xefK\xc1\xe5\x14\x9d\n\x17\xd6.\x1e\xc61w#_\vݽ\xf1\xa8S\x06\x13\x89I\x9bv\xa4s{\xc0\xce1\xa8%\x95\xe6]H\xb2ƿ\xc42L\x92\x82f2_R\u007f\xbe\x8dfy\x9c䗃\n8\xbd\x9c`zH2S\xff\x86v\xa8O\xda`1Q\xa6\t\xbe\r%\x1d\xb4\xb1\x9f\xfb\xac\xe1\x1e_\x16n\x1fإɚ\xe7\xfa\xf5\xb9Q\x1bP\xfe7{\xb2\xb3p\xa7\x91\x15\xa9\xfc\x0f\xbb\x1c\xd5\x17\x03z0\x04\x1c\xadM};\x9b\x90\x19\xc8t\x92\xcfdH\xb0_@\xb3\x88\xe7\xce\xee\\\xde\x04Tr\xac\xa4\xf4\x13\x0e\xc9\x1e\xfc\x14\\\v\x06o纜\xf9\xf0z\x17\xa1\xe5\xe4?\xe9\u007fSN\xe3\x86\x18\x17}\xd7\x19\xd5\xe2C\xf2\xb8\xc4\xf8r\u007f\r(\xa31jk\xb0\x05\xe18\xd7.\xba\x8aY\x9d\xa6U3\x96\xday\x9fz\xa3\x80f\n\xa9O^\x0ehou\x03\xbc\xa8锿\xf2\f\xdb\xd3-\xd5\xf5\xebr8o\xa9R\xba-\xa4\xd9]\v-p\xf6.R\x16\xb3WJzq\xf3\x98\x11\xb2\xb9\x94\x1dg\xc6Uk\x8c\x8b\xc8<\x86\x9b\x10\x16\x93=\xbb\xcc滋\xf0\x828V\xfb1\xe0\xf3\xe8M\x9b\x9a\x17\xec\xee\xa7+\xee\x87\x0fW\xbbj\xfe\xd4\xcevT6t\xf8\xf5\xb7\xaaX\xc5\xeei\\0\xd3\xe5\xdf\x01\x00\x00\xff\xff\xfb\xb1p\x12\x1b\f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WO\x8f۶\x13\xbd\xfbS\f\x92\xc3^\"9\xc1\xef\xf0+t)\x82M\x0fA\xf3g\x11o\xf7R\xf4@\x93#\x8b]\x8aTgHm\xddO_\f)\xad\xbd\xb6\x9cl\x8aV\x17C\x149|\xf3\u07bc!\xbd\xaa\xaaj\xa5\x06{\x87\xc46\xf8\x06\xd4`\xf1ψ^\u07b8\xbe\xff\x81k\x1b\xd6\xe3\x9bս\xf5\xa6\x81\xeb\xc41\xf4_\x90C\"\x8dﰵ\xdeF\x1b\xfc\xaaǨ\x8c\x8a\xaaY\x01(\xefCT2\xcc\xf2\n\xa0\x83\x8f\x14\x9cC\xaav\xe8\xeb\xfb\xb4\xc5m\xb2\xce \xe5\xe0\xf3\xd6\xe3\xeb\xfa\xff\xf5\xeb\x15\x80&\xcc\xcbom\x8f\x1cU?4\xe0\x93s+\x00\xafzl`\f.\xf5\xc8^\r܅\xe8\x82.\x9b\xd5#:\xa4P۰\xe2\x01\xb5콣\x90\x86\x06\x0e\x1fJ\x88\tW\xc9\xe9.G\xdbL\xd1>L\xd1\xf2\x04g9\xfe\xfc\x95I\x1f,\xc7\xe8\xcdK\x9a,\xcaWO\xb0ƽT\x11G\xb2~w\xf4!\x1b\xe1+\n\x88\aJ!\x94\xa5%\x8b\x03\xd12$\xec|\xf9is\v\xf3\xd6Y\x8cS\xf63\uf1c5|\x90@\b\xb3\xbeE*\"\xb6\x14\xfa\x1c\x13\xbd\x19\x82\xf51\xbfhgџ\xd2\xcfi\xdb\xdb(\xba\xff\x91\x90\xa3hU\xc3u\xeeB\xb0EH\x83Q\x11M\r\xef=\\\xab\x1eݵb\xfc\xcf\x05\x10\xa6\xb9\x12b\x9f'\xc1q\x03=\x9d\\X;6\xd8\xd4\xde.\xe8\xb5\xec\xe4̀\xfa\x89\x81$\x8am\xed\xe4\xec6\xd0\t\xafj\xf6\xf9r\xbc\xfa\xc9\xf4e\x83C\xe9\xfe\xadݝ\x8e\x02(c\xf2١\xdc\xcdŵ_!l!\xef뼓\x14j\x1bH\x10\x8d\xd6 Us\x9e\x13\x92DS\xc2\x16\x9d\xe1\xfa,\xe4\x05\xces*\x84F4V\xee\x1c\xe8S$\x8f\x13\xf3᧬/\x94\x1f\x02\xe4ң~\xea\xb1>\xa27\xb9\xa9\x9f\xa1\t\xb9\x86\x19\r<\xd8\xd8\x15s\xb8\xe3C\xeay*\xc8s\x8f\xfb\xa5\xe1\x13\xec\xb7\x1d\xca\xcc\xd2N\x11\x185a\x14\x1c\x8cN\xcc+ά\x01>&\xce\xf6R\x8b\x11AZ\x845\xf3\xea{ܟ\x13\r\xdf\x12w:\xef\xbf\r\xf9J\xce\xc5\x190a\x8b\x84>.Z\\\xee\x1e\xe41bv\xb9\t\x9a\xc5\xe0\x1a\x87\xc8\xeb0\"\x8d\x16\x1f\xd6\x0f\x81\xee\xad\xdfUBxU\n\x81\xd7\xf9ް~\x99\u007f.\xa4|\xfb\xf9\xdd\xe7\x06\xde\x1a\x03!vH\xa2Z\x9b\xdc\\hG\xa7ݫ\xdcq_A\xb2\xe6ǫ\u007f\xc2K\x18\x8as\x9e\xc1\xcd&W\xff^N\xee\fJ(\xda\x14U\x02\x81\xf4M\x11\xbb\x9f\xd4,\xfda\xa9\x10gL\xdb\x10\x1c\xaa\xf3ғ\xeek\t\xcd9\xa4Jv\xf8\x1e\x9b\xcd\xce\xfd\x86\xc9n\xa6ibx\xc9j^6\x17B\xb9\x97\xe4[\x8a\xda\xe1%\xa3/p\xbc\x9cJ\xf5\xb8\xc1\xb3ZtT1\xf1\xf77\xe9\xbcl\x9a\xb9\x9d\x1a\xb5N$\x05=\xc5\\\xb8\xd0\xfc;\x8dz\xe8\x14/\xb8\xed\x19\xa8od\xe5,\x83\xb3-\xea\xbdvX\x02Bh\x17\xaa\xe9\xbb ˃>\xf5K\xa5\xf5vT֩\xadÅo\xbfxu\xf1\xebE\xf1\x17\xf5<\x1bd\xb9\xb5\x98\x06\"\xa5\x12{\xaa\xb2i䠾\xd2\xd2\\\xd0|:\xfd\xdb\xf1\xe2œ\u007f\x0e\xf9U\a_\xceDn\xe0\xd7\xdfV%*\x9a\xbb\xf9\xa2/\x83\u007f\a\x00\x00\xff\xff\xe4\xf3S\x85\xb2\r\x00\x00"), } diff --git a/design/biav2-design.md b/design/biav2-design.md index a638b6b14d..ae178479fd 100644 --- a/design/biav2-design.md +++ b/design/biav2-design.md @@ -34,6 +34,7 @@ message ExecuteResponse { bytes item = 1; repeated generated.ResourceIdentifier additionalItems = 2; string operationID = 3; + repeated generated.ResourceIdentifier itemsToUpdate = 4; } ``` The BackupItemAction service gets two new rpc methods: @@ -78,6 +79,19 @@ message OperationProgress { } ``` +In addition to the two new rpc methods added to the BackupItemAction interface, there is also a new `Name()` method. This one is only actually used internally by Velero to get the name that the plugin was registered with, but it still must be defined in a plugin which implements BackupItemActionV2 in order to implement the interface. It doesn't really matter what it returns, though, as this particular method is not delegated to the plugin via RPC calls. The new (and modified) interface methods for `BackupItemAction` are as follows: +``` +type BackupItemAction interface { +... + Name() string +... + Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) + Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) + Cancel(operationID string, backup *api.Backup) error +... +} +``` + A new PluginKind, `BackupItemActionV2`, will be created, and the backup process will be modified to use this plugin kind. See [Plugin Versioning](plugin-versioning.md) for more details on implementation plans, including v1 adapters, etc. diff --git a/design/general-progress-monitoring.md b/design/general-progress-monitoring.md index cb627d822c..d5240c263e 100644 --- a/design/general-progress-monitoring.md +++ b/design/general-progress-monitoring.md @@ -113,13 +113,21 @@ long as they don't use not-yet-completed backups) to be made without interferenc all data has been moved before starting the next backup will slow the progress of the system without adding any actual benefit to the user. -A new backup/restore phase, "WaitingForPluginOperations" will be introduced. When a backup or -restore has entered this phase, Velero is free to start another backup/restore. The backup/restore +New backup/restore phases, "WaitingForPluginOperations" and +"WaitingForPluginOperationsPartiallyFailed" will be introduced. When a backup or restore has +entered one of these phases, Velero is free to start another backup/restore. The backup/restore will remain in the "WaitingForPluginOperations" phase until all BIA/RIA operations have completed (for example, for a volume snapshotter, until all data has been successfully moved to persistent -storage). The backup/restore will not fail once it reaches this phase. If the backup is deleted -(cancelled), the plug-ins will attempt to delete the snapshots and stop the data movement - this may -not be possible with all storage systems. +storage). The backup/restore will not fail once it reaches this phase, although an error return +from a plugin could cause a backup or restore to move to "PartiallyFailed". If the backup is +deleted (cancelled), the plug-ins will attempt to delete the snapshots and stop the data movement - +this may not be possible with all storage systems. + +In addition, for backups (but not restores), there will also be two additional phases, +"FinalizingAfterPluginOperations" and "FinalizingAfterPluginOperationsPartiallyFailed", which will +handle any steps required after plugin operations have all completed. Initially, this will just +include adding any required resources to the backup that might have changed during asynchronous +operation execution, although eventually other cleanup actions could be added to this phase. ### State progression @@ -143,7 +151,14 @@ In the current implementation, Restic backups will move data during the "InProgr future, it may be possible to combine a snapshot with a Restic (or equivalent) backup which would allow for data movement to be handled in the "WaitingForPluginOperations" phase, -The next phase is either "Completed", "WaitingForPluginOperations", "Failed" or "PartiallyFailed". +The next phase would be "WaitingForPluginOperations" for backups or restores which have unfinished +asynchronous plugin operations and no errors so far, "WaitingForPluginOperationsPartiallyFailed" for +backups or restores which have unfinished asynchronous plugin operations at least one error, +"Completed" for restores with no unfinished asynchronous plugin operations and no errors, +"PartiallyFailed" for restores with no unfinished asynchronous plugin operations and at least one +error, "FinalizingAfterPluginOperations" for backups with no unfinished asynchronous plugin +operations and no errors, "FinalizingAfterPluginOperationsPartiallyFailed" for backups with no +unfinished asynchronous plugin operations and at least one error, or "PartiallyFailed". Backups/restores which would have a final phase of "Completed" or "PartiallyFailed" may move to the "WaitingForPluginOperations" or "WaitingForPluginOperationsPartiallyFailed" state. A backup/restore which will be marked "Failed" will go directly to the "Failed" phase. Uploads may continue in the @@ -157,8 +172,9 @@ any uploads still in progress should be aborted. The "WaitingForPluginOperations" phase signifies that the main part of the backup/restore, including snapshotting has completed successfully and uploading and any other asynchronous BIA/RIA plugin operations are continuing. In the event of an error during this phase, the phase will change to -WaitingForPluginOperationsPartiallyFailed. On success, the phase changes to Completed. Backups -cannot be restored from when they are in the WaitingForPluginOperations state. +WaitingForPluginOperationsPartiallyFailed. On success, the phase changes to +"FinalizingAfterPluginOperations" for backups and "Completed" for restores. Backups cannot be +restored from when they are in the WaitingForPluginOperations state. ### WaitingForPluginOperationsPartiallyFailed (new) The "WaitingForPluginOperationsPartiallyFailed" phase signifies that the main part of the @@ -166,6 +182,22 @@ backup/restore, including snapshotting has completed, but there were partial fai the main part or during any async operations, including snapshot uploads. Backups cannot be restored from when they are in the WaitingForPluginOperationsPartiallyFailed state. +### FinalizingAfterPluginOperations (new) +The "FinalizingAfterPluginOperations" phase signifies that asynchronous backup operations have all +completed successfully and Velero is currently backing up any resources indicated by asynchronous +plugins as items to back up after operations complete. Once this is done, the phase changes to +Completed. Backups cannot be restored from when they are in the FinalizingAfterPluginOperations +state. + +### FinalizingAfterPluginOperationsPartiallyFailed (new) + +The "FinalizingAfterPluginOperationsPartiallyFailed" phase signifies that, for a backup which had +errors during initial processing or asynchronous plugin operation, asynchronous backup operations +have all completed and Velero is currently backing up any resources indicated by asynchronous +plugins as items to back up after operations complete. Once this is done, the phase changes to +PartiallyFailed. Backups cannot be restored from when they are in the +FinalizingAfterPluginOperationsPartiallyFailed state. + ### Failed When a backup/restore has had fatal errors it is marked as "Failed" Backups in this state cannot be restored from. @@ -211,12 +243,21 @@ WaitingForPluginOperationsPartiallyFailed phase, another backup/restore may be s While in the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase, the snapshots and item actions will be periodically polled. When all of the snapshots and item actions -have reported success, the backup/restore will move to the Completed or PartiallyFailed phase, -depending on whether the backup/restore was in the WaitingForPluginOperations or -WaitingForPluginOperationsPartiallyFailed phase. - -The Backup resources will not be written to object storage until the backup has entered a final phase: -Completed, Failed or PartiallyFailed +have reported success, restores will move directly to the Completed or PartiallyFailed phase, and +backups will move to the FinalizingAfterPluginOperations or +FinalizingAfterPluginOperationsPartiallyFailed phase, depending on whether the backup/restore was in +the WaitingForPluginOperations or WaitingForPluginOperationsPartiallyFailed phase. + +While in the FinalizingAfterPluginOperations or FinalizingAfterPluginOperationsPartiallyFailed +phase, Velero will update the backup with any resources indicated by plugins that they must be added +to the backup after operations are completed, and then the backup will move to the Completed or +PartiallyFailed phase, depending on whether there are any backup errors. + +The Backup resources will be written to object storage at the time the backup leaves the InProgress +phase, but it will not be synced to other clusters (or usable for restores in the current cluster) +until the backup has entered a final phase: Completed, Failed or PartiallyFailed. During the +Finalizing phases, a the backup resources will be updated with any required resources related to +asynchronous plugins. ## Reconciliation of InProgress backups @@ -249,8 +290,6 @@ Two new methods will be added to the VolumeSnapshotter interface: Progress(snapshotID string) (OperationProgress, error) Cancel(snapshotID string) (error) -Open question: Does VolumeSnapshotter need Cancel, or is that only needed for BIA/RIA? - Progress will report the current status of a snapshot upload. This should be callable at any time after the snapshot has been taken. In the event a plug-in is restarted, if the operationID (snapshot ID) continues to be valid it should be possible to retrieve the progress. @@ -281,35 +320,38 @@ progress, Progress will return an InvalidOperationIDError error rather than a po OperationProgress struct. If the item action does not start an asynchronous operation, then operationID will be empty. -Two new methods will be added to the BackupItemAction interface, and the Execute() return signature +Three new methods will be added to the BackupItemAction interface, and the Execute() return signature will be modified: - // Execute allows the ItemAction to perform arbitrary logic with the item being backed up, - // including mutating the item itself prior to backup. The item (unmodified or modified) - // should be returned, an optional operationID, along with an optional slice of ResourceIdentifiers - // specifying additional related items that should be backed up. If operationID is specified - // then velero will wait for this operation to complete before the backup is marked Completed. - Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, operationID string, - []ResourceIdentifier, error) + // Name returns the name of this BIA. Plugins which implement this interface must defined Name, + // but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure + // will implement this directly rather than delegating to the RPC plugin in order to return the name + // that the plugin was registered under. The plugins must implement the method to complete the interface. + Name() string + // Execute allows the BackupItemAction to perform arbitrary logic with the item being backed up, + // including mutating the item itself prior to backup. The item (unmodified or modified) + // should be returned, along with an optional slice of ResourceIdentifiers specifying + // additional related items that should be backed up now, an optional operationID for actions which + // initiate asynchronous actions, and a second slice of ResourceIdentifiers specifying related items + // which should be backed up after all asynchronous operations have completed. This last field will be + // ignored if operationID is empty, and should not be filled in unless the resource must be updated in the + // backup after async operations complete (i.e. some of the item's kubernetes metadata will be updated + // during the asynch operation which will be required during restore) + Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) // Progress - Progress(input *BackupItemActionProgressInput) (OperationProgress, error) + Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) // Cancel - Cancel(input *BackupItemActionProgressInput) error - - // BackupItemActionProgressInput contains the input parameters for the BackupItemAction's Progress function. - type BackupItemActionProgressInput struct { - // Item is the item that was stored in the backup - Item runtime.Unstructured - // OperationID is the operation ID returned by BackupItemAction Execute - operationID string - // Backup is the representation of the backup resource processed by Velero. - Backup *velerov1api.Backup - } + Cancel(operationID string, backup *api.Backup) error -Two new methods will be added to the RestoreItemAction interface, and the +Three new methods will be added to the RestoreItemAction interface, and the RestoreItemActionExecuteOutput struct will be modified: + // Name returns the name of this RIA. Plugins which implement this interface must defined Name, + // but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure + // will implement this directly rather than delegating to the RPC plugin in order to return the name + // that the plugin was registered under. The plugins must implement the method to complete the interface. + Name() string // Execute allows the ItemAction to perform arbitrary logic with the item being restored, // including mutating the item itself prior to restore. The item (unmodified or modified) // should be returned, an optional OperationID, along with an optional slice of ResourceIdentifiers @@ -321,10 +363,10 @@ RestoreItemActionExecuteOutput struct will be modified: // Progress - Progress(input *RestoreItemActionProgressInput) (OperationProgress, error) + Progress(operationID string, restore *api.Restore) (velero.OperationProgress, error) // Cancel - Cancel(input *RestoreItemActionProgressInput) error + Cancel(operationID string, restore *api.Restore) error // RestoreItemActionExecuteOutput contains the output variables for the ItemAction's Execution function. type RestoreItemActionExecuteOutput struct { @@ -345,16 +387,6 @@ RestoreItemActionExecuteOutput struct will be modified: OperationID string } - // RestoreItemActionProgressInput contains the input parameters for the RestoreItemAction's Progress function. - type RestoreItemActionProgressInput struct { - // Item is the item that was stored in the restore - Item runtime.Unstructured - // OperationID is the operation ID returned by RestoreItemAction Execute - operationID string - // Restore is the representation of the restore resource processed by Velero. - Restore *velerov1api.Restore - } - ## Changes in Velero backup format No changes to the existing format are introduced by this change. As part of the backup workflow changes, a @@ -370,19 +402,37 @@ to select the appropriate Backup/RestoreItemAction plugin to query for progress. of what a record for a datamover plugin might look like: ``` { - "itemOperation": { - "plugin": "velero.io/datamover-backup", - "itemID": "", - "operationID": "", - "completed": true, - "err": "", - "NCompleted": 12345, - "NTotal": 12345, - "OperationUnits": "byte", - "Description": "", - "Started": "2022-12-14T12:01:00Z", - "Updated": "2022-12-14T12:11:02Z" - } + "spec": { + "backupName": "backup-1", + "backupUID": "f8c72709-0f73-46e1-a071-116bc4a76b07", + "backupItemAction": "velero.io/volumesnapshotcontent-backup", + "resourceIdentifier": { + "Group": "snapshot.storage.k8s.io", + "Resource": "VolumeSnapshotContent", + "Namespace": "my-app", + "Name": "my-volume-vsc" + }, + "operationID": "", + "itemsToUpdate": [ + { + "Group": "velero.io", + "Resource": "VolumeSnapshotBackup", + "Namespace": "my-app", + "Name": "vsb-1" + } + ] + }, + "status": { + "operationPhase": "Completed", + "error": "", + "nCompleted": 12345, + "nTotal": 12345, + "operationUnits": "byte", + "description": "", + "Created": "2022-12-14T12:00:00Z", + "Started": "2022-12-14T12:01:00Z", + "Updated": "2022-12-14T12:11:02Z" + }, } ``` @@ -425,36 +475,41 @@ progress. ## Backup workflow changes The backup workflow remains the same until we get to the point where the `velero-backup.json` object -is written. At this point, we will queue the backup to a finalization go-routine. The next backup -may then begin. The finalization routine will run across all of the -VolumeSnapshotter/BackupItemAction operations and call the _Progress_ method on each of them. +is written. At this point, Velero will +run across all of the VolumeSnapshotter/BackupItemAction operations and call the _Progress_ method +on each of them. -If all snapshot and backup item operations have finished (either successfully or failed), the backup -will be completed and the backup will move to the appropriate terminal phase and upload the -`velero-backup.json` object to the object store and the backup will be complete. +If all backup item operations have finished (either successfully or failed), the backup will move to +one of the finalize phases. If any of the snapshots or backup items are still being processed, the phase of the backup will be set to the appropriate phase (_WaitingForPluginOperations_ or -_WaitingForPluginOperationsPartiallyFailed_). In the event of any of the progress checks return an -error, the phase will move to _WaitingForPluginOperationsPartiallyFailed_. The backup will then be -requeued and will be rechecked again after some time has passed. +_WaitingForPluginOperationsPartiallyFailed_), and the async backup operations controller will +reconcile periodically and call Progress on any unfinished operations. In the event of any of the +progress checks return an error, the phase will move to _WaitingForPluginOperationsPartiallyFailed_. + +Once all operations have completed, the backup will be moved to one of the finalize phases, and the +backup finalizer controller will update the the `velero-backup.json`in the object store with any +resources necessary after asynchronous operations are complete and the backup will move to the +appropriate terminal phase. + ## Restore workflow changes The restore workflow remains the same until velero would currently move the backup into one of the -terminal states. At this point, we will queue the restore to a finalization go-routine. The next -restore may then begin. The finalization routine will run across all of the RestoreItemAction -operations and call the _Progress_ method on each of them. +terminal states. At this point, Velero will run across all of the RestoreItemAction operations and +call the _Progress_ method on each of them. If all restore item operations have finished (either successfully or failed), the restore will be completed and the restore will move to the appropriate terminal phase and the restore will be complete. If any of the restore items are still being processed, the phase of the restore will be set to the -appropriate phase (_WaitingForPluginOperations_ or _WaitingForPluginOperationsPartiallyFailed_). In -the event of any of the progress checks return an error, the phase will move to -_WaitingForPluginOperationsPartiallyFailed_. The restore will then be requeued and will be rechecked -again after some time has passed. +appropriate phase (_WaitingForPluginOperations_ or _WaitingForPluginOperationsPartiallyFailed_), and +the async restore operations controller will reconcile periodically and call Progress on any +unfinished operations. In the event of any of the progress checks return an error, the phase will +move to _WaitingForPluginOperationsPartiallyFailed_. Once all of the operations have completed, the +restore will be moved to the appropriate terminal phase. ## Restart workflow diff --git a/internal/hook/item_hook_handler.go b/internal/hook/item_hook_handler.go index 8c3738050c..50ab84b2d3 100644 --- a/internal/hook/item_hook_handler.go +++ b/internal/hook/item_hook_handler.go @@ -273,6 +273,20 @@ func (h *DefaultItemHookHandler) HandleHooks( return nil } +// NoOpItemHookHandler is the an itemHookHandler for the Finalize controller where hooks don't run +type NoOpItemHookHandler struct{} + +func (h *NoOpItemHookHandler) HandleHooks( + log logrus.FieldLogger, + groupResource schema.GroupResource, + obj runtime.Unstructured, + resourceHooks []ResourceHook, + phase hookPhase, +) error { + + return nil +} + func phasedKey(phase hookPhase, key string) string { if phase != "" { return fmt.Sprintf("%v.%v", phase, key) diff --git a/pkg/apis/velero/v1/backup.go b/pkg/apis/velero/v1/backup.go index 4c43784f00..394fe969c9 100644 --- a/pkg/apis/velero/v1/backup.go +++ b/pkg/apis/velero/v1/backup.go @@ -124,6 +124,11 @@ type BackupSpec struct { // The default value is 10 minute. // +optional CSISnapshotTimeout metav1.Duration `json:"csiSnapshotTimeout,omitempty"` + + // ItemOperationTimeout specifies the time used to wait for asynchronous BackupItemAction operations + // The default value is 1 hour. + // +optional + ItemOperationTimeout metav1.Duration `json:"itemOperationTimeout,omitempty"` } // BackupHooks contains custom behaviors that should be executed at different phases of the backup. @@ -221,7 +226,7 @@ const ( // BackupPhase is a string representation of the lifecycle phase // of a Velero backup. -// +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting +// +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;FinalizingAfterPluginOperations;FinalizingAfterPluginOperationsPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting type BackupPhase string const ( @@ -251,6 +256,23 @@ const ( // ongoing. The backup is not usable yet. BackupPhaseWaitingForPluginOperationsPartiallyFailed BackupPhase = "WaitingForPluginOperationsPartiallyFailed" + // BackupPhaseFinalizingAfterPluginOperations means the backup of + // Kubernetes resources, creation of snapshots, and other + // async plugin operations were successful and snapshot upload and + // other plugin operations are now complete, but the Backup is awaiting + // final update of resources modified during async operations. + // The backup is not usable yet. + BackupPhaseFinalizingAfterPluginOperations BackupPhase = "FinalizingAfterPluginOperations" + + // BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed means the backup of + // Kubernetes resources, creation of snapshots, and other + // async plugin operations were successful and snapshot upload and + // other plugin operations are now complete, but one or more errors + // occurred during backup or async operation processing, and the + // Backup is awaiting final update of resources modified during async + // operations. The backup is not usable yet. + BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed BackupPhase = "FinalizingAfterPluginOperationsPartiallyFailed" + // BackupPhaseCompleted means the backup has run successfully without // errors. BackupPhaseCompleted BackupPhase = "Completed" @@ -351,6 +373,21 @@ type BackupStatus struct { // completed CSI VolumeSnapshots for this backup. // +optional CSIVolumeSnapshotsCompleted int `json:"csiVolumeSnapshotsCompleted,omitempty"` + + // AsyncBackupItemOperationsAttempted is the total number of attempted + // async BackupItemAction operations for this backup. + // +optional + AsyncBackupItemOperationsAttempted int `json:"asyncBackupItemOperationsAttempted,omitempty"` + + // AsyncBackupItemOperationsCompleted is the total number of successfully completed + // async BackupItemAction operations for this backup. + // +optional + AsyncBackupItemOperationsCompleted int `json:"asyncBackupItemOperationsCompleted,omitempty"` + + // AsyncBackupItemOperationsFailed is the total number of async + // BackupItemAction operations for this backup which ended with an error. + // +optional + AsyncBackupItemOperationsFailed int `json:"asyncBackupItemOperationsFailed,omitempty"` } // BackupProgress stores information about the progress of a Backup's execution. diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 8c488dd4f6..71ec427ed3 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -350,6 +350,7 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { } } out.CSISnapshotTimeout = in.CSISnapshotTimeout + out.ItemOperationTimeout = in.ItemOperationTimeout } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec. diff --git a/pkg/archive/filesystem.go b/pkg/archive/filesystem.go index 32eed2bd29..6c4feeb3f2 100644 --- a/pkg/archive/filesystem.go +++ b/pkg/archive/filesystem.go @@ -28,11 +28,20 @@ import ( // GetItemFilePath returns an item's file path once extracted from a Velero backup archive. func GetItemFilePath(rootDir, groupResource, namespace, name string) string { - switch namespace { - case "": - return filepath.Join(rootDir, velerov1api.ResourcesDir, groupResource, velerov1api.ClusterScopedDir, name+".json") - default: - return filepath.Join(rootDir, velerov1api.ResourcesDir, groupResource, velerov1api.NamespaceScopedDir, namespace, name+".json") + return GetVersionedItemFilePath(rootDir, groupResource, namespace, name, "") +} + +// GetVersionedItemFilePath returns an item's file path once extracted from a Velero backup archive, with version included. +func GetVersionedItemFilePath(rootDir, groupResource, namespace, name, versionPath string) string { + return filepath.Join(rootDir, velerov1api.ResourcesDir, groupResource, versionPath, GetScopeDir(namespace), namespace, name+".json") +} + +// GetScopeDir returns NamespaceScopedDir if namespace is present, or ClusterScopedDir if empty +func GetScopeDir(namespace string) string { + if namespace == "" { + return velerov1api.ClusterScopedDir + } else { + return velerov1api.NamespaceScopedDir } } diff --git a/pkg/archive/filesystem_test.go b/pkg/archive/filesystem_test.go index bb07af928c..85632fa504 100644 --- a/pkg/archive/filesystem_test.go +++ b/pkg/archive/filesystem_test.go @@ -28,4 +28,22 @@ func TestGetItemFilePath(t *testing.T) { res = GetItemFilePath("root", "resource", "namespace", "item") assert.Equal(t, "root/resources/resource/namespaces/namespace/item.json", res) + + res = GetItemFilePath("", "resource", "", "item") + assert.Equal(t, "resources/resource/cluster/item.json", res) + + res = GetVersionedItemFilePath("root", "resource", "", "item", "") + assert.Equal(t, "root/resources/resource/cluster/item.json", res) + + res = GetVersionedItemFilePath("root", "resource", "namespace", "item", "") + assert.Equal(t, "root/resources/resource/namespaces/namespace/item.json", res) + + res = GetVersionedItemFilePath("root", "resource", "namespace", "item", "v1") + assert.Equal(t, "root/resources/resource/v1/namespaces/namespace/item.json", res) + + res = GetVersionedItemFilePath("root", "resource", "", "item", "v1") + assert.Equal(t, "root/resources/resource/v1/cluster/item.json", res) + + res = GetVersionedItemFilePath("", "resource", "", "item", "") + assert.Equal(t, "resources/resource/cluster/item.json", res) } diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 9db76b1b57..ab3d8ecc35 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -42,8 +42,10 @@ import ( "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podexec" @@ -67,6 +69,9 @@ type Backupper interface { BackupWithResolvers(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, backupItemActionResolver framework.BackupItemActionResolverV2, itemSnapshotterResolver framework.ItemSnapshotterResolver, volumeSnapshotterGetter VolumeSnapshotterGetter) error + FinalizeBackup(log logrus.FieldLogger, backupRequest *Request, inBackupFile io.Reader, outBackupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolverV2, + asyncBIAOperations []*itemoperation.BackupOperation) error } // kubernetesBackupper implements Backupper. @@ -434,6 +439,25 @@ func (kb *kubernetesBackupper) backupItem(log logrus.FieldLogger, gr schema.Grou return backedUpItem } +func (kb *kubernetesBackupper) finalizeItem(log logrus.FieldLogger, gr schema.GroupResource, itemBackupper *itemBackupper, unstructured *unstructured.Unstructured, preferredGVR schema.GroupVersionResource) (bool, []FileForArchive) { + backedUpItem, updateFiles, err := itemBackupper.finalizeItem(log, unstructured, gr, preferredGVR) + if aggregate, ok := err.(kubeerrs.Aggregate); ok { + log.WithField("name", unstructured.GetName()).Infof("%d errors encountered backup up item", len(aggregate.Errors())) + // log each error separately so we get error location info in the log, and an + // accurate count of errors + for _, err = range aggregate.Errors() { + log.WithError(err).WithField("name", unstructured.GetName()).Error("Error backing up item") + } + + return false, updateFiles + } + if err != nil { + log.WithError(err).WithField("name", unstructured.GetName()).Error("Error backing up item") + return false, updateFiles + } + return backedUpItem, updateFiles +} + // backupCRD checks if the resource is a custom resource, and if so, backs up the custom resource definition // associated with it. func (kb *kubernetesBackupper) backupCRD(log logrus.FieldLogger, gr schema.GroupResource, itemBackupper *itemBackupper) { @@ -492,6 +516,180 @@ func (kb *kubernetesBackupper) writeBackupVersion(tw *tar.Writer) error { return nil } +func (kb *kubernetesBackupper) FinalizeBackup(log logrus.FieldLogger, + backupRequest *Request, + inBackupFile io.Reader, + outBackupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolverV2, + asyncBIAOperations []*itemoperation.BackupOperation) error { + + gzw := gzip.NewWriter(outBackupFile) + defer gzw.Close() + tw := tar.NewWriter(gzw) + defer tw.Close() + + gzr, err := gzip.NewReader(inBackupFile) + if err != nil { + log.Infof("error creating gzip reader: %v", err) + return err + } + defer gzr.Close() + tr := tar.NewReader(gzr) + + backupRequest.ResolvedActions, err = backupItemActionResolver.ResolveActions(kb.discoveryHelper, log) + if err != nil { + log.WithError(errors.WithStack(err)).Debugf("Error from backupItemActionResolver.ResolveActions") + return err + } + + backupRequest.BackedUpItems = map[itemKey]struct{}{} + + // set up a temp dir for the itemCollector to use to temporarily + // store items as they're scraped from the API. + tempDir, err := ioutil.TempDir("", "") + if err != nil { + return errors.Wrap(err, "error creating temp dir for backup") + } + defer os.RemoveAll(tempDir) + + collector := &itemCollector{ + log: log, + backupRequest: backupRequest, + discoveryHelper: kb.discoveryHelper, + dynamicFactory: kb.dynamicFactory, + cohabitatingResources: cohabitatingResources(), + dir: tempDir, + pageSize: kb.clientPageSize, + } + + // Get item list from itemoperation.BackupOperation.Spec.ItemsToUpdate + var resourceIDs []velero.ResourceIdentifier + for _, operation := range asyncBIAOperations { + if len(operation.Spec.ItemsToUpdate) != 0 { + resourceIDs = append(resourceIDs, operation.Spec.ItemsToUpdate...) + } + } + items := collector.getItemsFromResourceIdentifiers(resourceIDs) + log.WithField("progress", "").Infof("Collected %d items from the async BIA operations ItemsToUpdate list", len(items)) + + itemBackupper := &itemBackupper{ + backupRequest: backupRequest, + tarWriter: tw, + dynamicFactory: kb.dynamicFactory, + discoveryHelper: kb.discoveryHelper, + itemHookHandler: &hook.NoOpItemHookHandler{}, + } + updateFiles := make(map[string]FileForArchive) + backedUpGroupResources := map[schema.GroupResource]bool{} + totalItems := len(items) + + for i, item := range items { + log.WithFields(map[string]interface{}{ + "progress": "", + "resource": item.groupResource.String(), + "namespace": item.namespace, + "name": item.name, + }).Infof("Processing item") + + // use an anonymous func so we can defer-close/remove the file + // as soon as we're done with it + func() { + var unstructured unstructured.Unstructured + + f, err := os.Open(item.path) + if err != nil { + log.WithError(errors.WithStack(err)).Error("Error opening file containing item") + return + } + defer f.Close() + defer os.Remove(f.Name()) + + if err := json.NewDecoder(f).Decode(&unstructured); err != nil { + log.WithError(errors.WithStack(err)).Error("Error decoding JSON from file") + return + } + + backedUp, itemFiles := kb.finalizeItem(log, item.groupResource, itemBackupper, &unstructured, item.preferredGVR) + if backedUp { + backedUpGroupResources[item.groupResource] = true + for _, itemFile := range itemFiles { + updateFiles[itemFile.FilePath] = itemFile + } + } + + }() + + // updated total is computed as "how many items we've backed up so far, plus + // how many items we know of that are remaining" + totalItems = len(backupRequest.BackedUpItems) + (len(items) - (i + 1)) + + log.WithFields(map[string]interface{}{ + "progress": "", + "resource": item.groupResource.String(), + "namespace": item.namespace, + "name": item.name, + }).Infof("Updated %d items out of an estimated total of %d (estimate will change throughout the backup finalizer)", len(backupRequest.BackedUpItems), totalItems) + } + + // write new tar archive replacing files in original with content updateFiles for matches + buildFinalTarball(tr, tw, updateFiles) + log.WithField("progress", "").Infof("Updated a total of %d items", len(backupRequest.BackedUpItems)) + + return nil +} + +func buildFinalTarball(tr *tar.Reader, tw *tar.Writer, updateFiles map[string]FileForArchive) error { + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return errors.WithStack(err) + } + newFile, ok := updateFiles[header.Name] + if ok { + // add updated file to archive, skip over tr file content + if err := tw.WriteHeader(newFile.Header); err != nil { + return errors.WithStack(err) + } + if _, err := tw.Write(newFile.FileBytes); err != nil { + return errors.WithStack(err) + } + delete(updateFiles, header.Name) + // skip over file contents from old tarball + _, err := io.ReadAll(tr) + if err != nil { + return errors.WithStack(err) + } + } else { + // Add original content to new tarball, as item wasn't updated + oldContents, err := io.ReadAll(tr) + if err != nil { + return errors.WithStack(err) + } + if err := tw.WriteHeader(header); err != nil { + return errors.WithStack(err) + } + if _, err := tw.Write(oldContents); err != nil { + return errors.WithStack(err) + } + } + } + // iterate over any remaining map entries, which represent updated items that + // were not in the original backup tarball + for _, newFile := range updateFiles { + if err := tw.WriteHeader(newFile.Header); err != nil { + return errors.WithStack(err) + } + if _, err := tw.Write(newFile.FileBytes); err != nil { + return errors.WithStack(err) + } + } + return nil + +} + type tarWriter interface { io.Closer Write([]byte) (int, error) diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 5b37ff1ba9..9006400174 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -45,6 +45,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" @@ -1140,17 +1141,18 @@ type recordResourcesAction struct { backups []velerov1.Backup additionalItems []velero.ResourceIdentifier operationID string + itemsToUpdate []velero.ResourceIdentifier } -func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { metadata, err := meta.Accessor(item) if err != nil { - return item, a.additionalItems, a.operationID, err + return item, a.additionalItems, a.operationID, a.itemsToUpdate, err } a.ids = append(a.ids, kubeutil.NamespaceAndName(metadata)) a.backups = append(a.backups, *backup) - return item, a.additionalItems, a.operationID, nil + return item, a.additionalItems, a.operationID, a.itemsToUpdate, nil } func (a *recordResourcesAction) AppliesTo() (velero.ResourceSelector, error) { @@ -1165,6 +1167,10 @@ func (a *recordResourcesAction) Cancel(operationID string, backup *velerov1.Back return nil } +func (a *recordResourcesAction) Name() string { + return "" +} + func (a *recordResourcesAction) ForResource(resource string) *recordResourcesAction { a.selector.IncludedResources = append(a.selector.IncludedResources, resource) return a @@ -1462,7 +1468,7 @@ func (a *appliesToErrorAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{}, errors.New("error calling AppliesTo") } -func (a *appliesToErrorAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (a *appliesToErrorAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { panic("not implemented") } @@ -1474,6 +1480,10 @@ func (a *appliesToErrorAction) Cancel(operationID string, backup *velerov1.Backu panic("not implemented") } +func (a *appliesToErrorAction) Name() string { + return "" +} + // TestBackupActionModifications runs backups with backup item actions that make modifications // to items in their Execute(...) methods and verifies that these modifications are // persisted to the backup tarball. Verification is done by inspecting the file contents @@ -1483,16 +1493,16 @@ func TestBackupActionModifications(t *testing.T) { // method modifies the item being passed in by calling the 'modify' function on it. modifyingActionGetter := func(modify func(*unstructured.Unstructured)) *pluggableAction { return &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { obj, ok := item.(*unstructured.Unstructured) if !ok { - return nil, nil, "", errors.Errorf("unexpected type %T", item) + return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) } res := obj.DeepCopy() modify(res) - return res, nil, "", nil + return res, nil, "", nil, nil }, } } @@ -1621,13 +1631,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, {GroupResource: kuberesource.Pods, Namespace: "ns-3", Name: "pod-3"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1652,13 +1662,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, {GroupResource: kuberesource.Pods, Namespace: "ns-3", Name: "pod-3"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1682,13 +1692,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1715,13 +1725,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1745,13 +1755,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1776,13 +1786,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1807,13 +1817,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-4", Name: "pod-4"}, {GroupResource: kuberesource.Pods, Namespace: "ns-5", Name: "pod-5"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -2292,6 +2302,167 @@ func TestBackupWithSnapshots(t *testing.T) { } } +// TestBackupWithAsyncOperations runs backups which return operationIDs and +// verifies that the itemoperations are tracked as appropriate. Verification is done by +// looking at the backup request's itemOperationsList field. +func TestBackupWithAsyncOperations(t *testing.T) { + // completedOperationAction is a *pluggableAction, whose Execute(...) + // method returns an operationID which will always be done when calling Progress. + completedOperationAction := &pluggableAction{ + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + obj, ok := item.(*unstructured.Unstructured) + if !ok { + return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) + } + + return obj, nil, obj.GetName() + "-1", nil, nil + }, + progressFunc: func(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { + return velero.OperationProgress{ + Completed: true, + Description: "Done!", + }, nil + }, + } + + // incompleteOperationAction is a *pluggableAction, whose Execute(...) + // method returns an operationID which will never be done when calling Progress. + incompleteOperationAction := &pluggableAction{ + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + obj, ok := item.(*unstructured.Unstructured) + if !ok { + return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) + } + + return obj, nil, obj.GetName() + "-1", nil, nil + }, + progressFunc: func(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { + return velero.OperationProgress{ + Completed: false, + Description: "Working...", + }, nil + }, + } + + // noOperationAction is a *pluggableAction, whose Execute(...) + // method does not return an operationID. + noOperationAction := &pluggableAction{ + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + obj, ok := item.(*unstructured.Unstructured) + if !ok { + return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) + } + + return obj, nil, "", nil, nil + }, + } + + tests := []struct { + name string + req *Request + apiResources []*test.APIResource + actions []biav2.BackupItemAction + want []*itemoperation.BackupOperation + }{ + { + name: "action that starts a short-running process records operation", + req: &Request{ + Backup: defaultBackup().Result(), + }, + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + ), + }, + actions: []biav2.BackupItemAction{ + completedOperationAction, + }, + want: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-1", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1"}, + OperationID: "pod-1-1", + }, + Status: itemoperation.OperationStatus{ + Phase: "InProgress", + }, + }, + }, + }, + { + name: "action that starts a long-running process records operation", + req: &Request{ + Backup: defaultBackup().Result(), + }, + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-2").Result(), + ), + }, + actions: []biav2.BackupItemAction{ + incompleteOperationAction, + }, + want: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-1", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-2"}, + OperationID: "pod-2-1", + }, + Status: itemoperation.OperationStatus{ + Phase: "InProgress", + }, + }, + }, + }, + { + name: "action that has no operation doesn't record one", + req: &Request{ + Backup: defaultBackup().Result(), + }, + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-3").Result(), + ), + }, + actions: []biav2.BackupItemAction{ + noOperationAction, + }, + want: []*itemoperation.BackupOperation{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var ( + h = newHarness(t) + backupFile = bytes.NewBuffer([]byte{}) + ) + + for _, resource := range tc.apiResources { + h.addItems(t, resource) + } + + err := h.backupper.Backup(h.log, tc.req, backupFile, tc.actions, nil) + assert.NoError(t, err) + + resultOper := *tc.req.GetItemOperationsList() + // set want Created times so it won't fail the assert.Equal test + for i, wantOper := range tc.want { + wantOper.Status.Created = resultOper[i].Status.Created + } + assert.Equal(t, tc.want, *tc.req.GetItemOperationsList()) + }) + } +} + // TestBackupWithInvalidHooks runs backups with invalid hook specifications and verifies // that an error is returned. func TestBackupWithInvalidHooks(t *testing.T) { @@ -2767,16 +2938,17 @@ func TestBackupWithPodVolume(t *testing.T) { } } -// pluggableAction is a backup item action that can be plugged with an Execute -// function body at runtime. +// pluggableAction is a backup item action that can be plugged with Execute +// and Progress function bodies at runtime. type pluggableAction struct { - selector velero.ResourceSelector - executeFunc func(runtime.Unstructured, *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) + selector velero.ResourceSelector + executeFunc func(runtime.Unstructured, *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) + progressFunc func(string, *velerov1.Backup) (velero.OperationProgress, error) } -func (a *pluggableAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (a *pluggableAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { if a.executeFunc == nil { - return item, nil, "", nil + return item, nil, "", nil, nil } return a.executeFunc(item, backup) @@ -2787,13 +2959,21 @@ func (a *pluggableAction) AppliesTo() (velero.ResourceSelector, error) { } func (a *pluggableAction) Progress(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { - return velero.OperationProgress{}, nil + if a.progressFunc == nil { + return velero.OperationProgress{}, nil + } + + return a.progressFunc(operationID, backup) } func (a *pluggableAction) Cancel(operationID string, backup *velerov1.Backup) error { return nil } +func (a *pluggableAction) Name() string { + return "" +} + type harness struct { *test.APIServer backupper *kubernetesBackupper diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 85d2766c79..4bef0f5cf7 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -20,7 +20,6 @@ import ( "archive/tar" "encoding/json" "fmt" - "path/filepath" "strings" "time" @@ -39,10 +38,13 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/util/boolptr" @@ -68,52 +70,87 @@ type itemBackupper struct { snapshotLocationVolumeSnapshotters map[string]vsv1.VolumeSnapshotter } +type FileForArchive struct { + FilePath string + Header *tar.Header + FileBytes []byte +} + +// finalizeItem backs up an individual item and returns its content to replace previous content +// in the backup tarball +// In addition to the error return, backupItem also returns a bool indicating whether the item +// was actually backed up and a slice of filepaths and filecontent to replace the data in the original tarball. +func (ib *itemBackupper) finalizeItem(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource) (bool, []FileForArchive, error) { + return ib.backupItemInternal(logger, obj, groupResource, preferredGVR, true, true) +} + // backupItem backs up an individual item to tarWriter. The item may be excluded based on the // namespaces IncludesExcludes list. // In addition to the error return, backupItem also returns a bool indicating whether the item // was actually backed up. func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource, mustInclude bool) (bool, error) { + selectedForBackup, files, err := ib.backupItemInternal(logger, obj, groupResource, preferredGVR, mustInclude, false) + // return if not selected, an error occurred, or there are no files to add + if selectedForBackup == false || err != nil || len(files) == 0 { + return selectedForBackup, err + } + for _, file := range files { + if err := ib.tarWriter.WriteHeader(file.Header); err != nil { + return false, errors.WithStack(err) + } + + if _, err := ib.tarWriter.Write(file.FileBytes); err != nil { + return false, errors.WithStack(err) + } + } + return true, nil +} + +func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runtime.Unstructured, groupResource schema.GroupResource, preferredGVR schema.GroupVersionResource, mustInclude, finalize bool) (bool, []FileForArchive, error) { + var itemFiles []FileForArchive metadata, err := meta.Accessor(obj) if err != nil { - return false, err + return false, itemFiles, err } namespace := metadata.GetNamespace() name := metadata.GetName() - log := logger.WithField("name", name) - log = log.WithField("resource", groupResource.String()) - log = log.WithField("namespace", namespace) + log := logger.WithFields(map[string]interface{}{ + "name": name, + "resource": groupResource.String(), + "namespace": namespace, + }) if mustInclude { log.Infof("Skipping the exclusion checks for this resource") } else { if metadata.GetLabels()[excludeFromBackupLabel] == "true" { log.Infof("Excluding item because it has label %s=true", excludeFromBackupLabel) - return false, nil + return false, itemFiles, nil } // NOTE: we have to re-check namespace & resource includes/excludes because it's possible that // backupItem can be invoked by a custom action. if namespace != "" && !ib.backupRequest.NamespaceIncludesExcludes.ShouldInclude(namespace) { log.Info("Excluding item because namespace is excluded") - return false, nil + return false, itemFiles, nil } // NOTE: we specifically allow namespaces to be backed up even if IncludeClusterResources is // false. if namespace == "" && groupResource != kuberesource.Namespaces && ib.backupRequest.Spec.IncludeClusterResources != nil && !*ib.backupRequest.Spec.IncludeClusterResources { log.Info("Excluding item because resource is cluster-scoped and backup.spec.includeClusterResources is false") - return false, nil + return false, itemFiles, nil } if !ib.backupRequest.ResourceIncludesExcludes.ShouldInclude(groupResource.String()) { log.Info("Excluding item because resource is excluded") - return false, nil + return false, itemFiles, nil } } if metadata.GetDeletionTimestamp() != nil { log.Info("Skipping item because it's being deleted.") - return false, nil + return false, itemFiles, nil } key := itemKey{ @@ -125,24 +162,23 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr if _, exists := ib.backupRequest.BackedUpItems[key]; exists { log.Info("Skipping item because it's already been backed up.") // returning true since this item *is* in the backup, even though we're not backing it up here - return true, nil + return true, itemFiles, nil } ib.backupRequest.BackedUpItems[key] = struct{}{} - log.Info("Backing up item") - log.Debug("Executing pre hooks") - if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePre); err != nil { - return false, err - } - var ( backupErrs []error pod *corev1api.Pod pvbVolumes []string ) - if groupResource == kuberesource.Pods { + log.Debug("Executing pre hooks") + if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePre); err != nil { + return false, itemFiles, err + } + + if !finalize && groupResource == kuberesource.Pods { // pod needs to be initialized for the unstructured converter pod = new(corev1api.Pod) if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pod); err != nil { @@ -166,7 +202,6 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr }).Info("Pod volume uses a persistent volume claim which has already been backed up from another pod, skipping.") continue } - pvbVolumes = append(pvbVolumes, volume) } } @@ -174,11 +209,9 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr // capture the version of the object before invoking plugin actions as the plugin may update // the group version of the object. - // group version of this object - // Used on filepath to backup up all groups and versions - version := resourceVersion(obj) + versionPath := resourceVersion(obj) - updatedObj, err := ib.executeActions(log, obj, groupResource, name, namespace, metadata) + updatedObj, err := ib.executeActions(log, obj, groupResource, name, namespace, metadata, finalize) if err != nil { backupErrs = append(backupErrs, err) @@ -187,24 +220,23 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost); err != nil { backupErrs = append(backupErrs, err) } - - return false, kubeerrs.NewAggregate(backupErrs) + return false, itemFiles, kubeerrs.NewAggregate(backupErrs) } obj = updatedObj if metadata, err = meta.Accessor(obj); err != nil { - return false, errors.WithStack(err) + return false, itemFiles, errors.WithStack(err) } // update name and namespace in case they were modified in an action name = metadata.GetName() namespace = metadata.GetNamespace() - if groupResource == kuberesource.PersistentVolumes { + if !finalize && groupResource == kuberesource.PersistentVolumes { if err := ib.takePVSnapshot(obj, log); err != nil { backupErrs = append(backupErrs, err) } } - if groupResource == kuberesource.Pods && pod != nil { + if !finalize && groupResource == kuberesource.Pods && pod != nil { // this function will return partial results, so process podVolumeBackups // even if there are errors. podVolumeBackups, errs := ib.backupPodVolumes(log, pod, pvbVolumes) @@ -224,33 +256,27 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr } if len(backupErrs) != 0 { - return false, kubeerrs.NewAggregate(backupErrs) + return false, itemFiles, kubeerrs.NewAggregate(backupErrs) } - // Getting the preferred group version of this resource - preferredVersion := preferredGVR.Version - - var filePath string - - // API Group version is now part of path of backup as a subdirectory - // it will add a prefix to subdirectory name for the preferred version - versionPath := version - - if version == preferredVersion { - versionPath = version + velerov1api.PreferredVersionDir + itemBytes, err := json.Marshal(obj.UnstructuredContent()) + if err != nil { + return false, itemFiles, errors.WithStack(err) } - if namespace != "" { - filePath = filepath.Join(velerov1api.ResourcesDir, groupResource.String(), versionPath, velerov1api.NamespaceScopedDir, namespace, name+".json") - } else { - filePath = filepath.Join(velerov1api.ResourcesDir, groupResource.String(), versionPath, velerov1api.ClusterScopedDir, name+".json") + if versionPath == preferredGVR.Version { + // backing up preferred version backup without API Group version - for backward compatibility + log.Debugf("Resource %s/%s, version= %s, preferredVersion=%s", groupResource.String(), name, versionPath, preferredGVR.Version) + itemFiles = append(itemFiles, getFileForArchive(namespace, name, groupResource.String(), "", itemBytes)) + versionPath = versionPath + velerov1api.PreferredVersionDir } - itemBytes, err := json.Marshal(obj.UnstructuredContent()) - if err != nil { - return false, errors.WithStack(err) - } + itemFiles = append(itemFiles, getFileForArchive(namespace, name, groupResource.String(), versionPath, itemBytes)) + return true, itemFiles, nil +} +func getFileForArchive(namespace, name, groupResource, versionPath string, itemBytes []byte) FileForArchive { + filePath := archive.GetVersionedItemFilePath("", groupResource, namespace, name, versionPath) hdr := &tar.Header{ Name: filePath, Size: int64(len(itemBytes)), @@ -258,43 +284,7 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr Mode: 0755, ModTime: time.Now(), } - - if err := ib.tarWriter.WriteHeader(hdr); err != nil { - return false, errors.WithStack(err) - } - - if _, err := ib.tarWriter.Write(itemBytes); err != nil { - return false, errors.WithStack(err) - } - - // backing up the preferred version backup without API Group version on path - this is for backward compatibility - - log.Debugf("Resource %s/%s, version= %s, preferredVersion=%s", groupResource.String(), name, version, preferredVersion) - if version == preferredVersion { - if namespace != "" { - filePath = filepath.Join(velerov1api.ResourcesDir, groupResource.String(), velerov1api.NamespaceScopedDir, namespace, name+".json") - } else { - filePath = filepath.Join(velerov1api.ResourcesDir, groupResource.String(), velerov1api.ClusterScopedDir, name+".json") - } - - hdr = &tar.Header{ - Name: filePath, - Size: int64(len(itemBytes)), - Typeflag: tar.TypeReg, - Mode: 0755, - ModTime: time.Now(), - } - - if err := ib.tarWriter.WriteHeader(hdr); err != nil { - return false, errors.WithStack(err) - } - - if _, err := ib.tarWriter.Write(itemBytes); err != nil { - return false, errors.WithStack(err) - } - } - - return true, nil + return FileForArchive{FilePath: filePath, Header: hdr, FileBytes: itemBytes} } // backupPodVolumes triggers pod volume backups of the specified pod volumes, and returns a list of PodVolumeBackups @@ -318,6 +308,7 @@ func (ib *itemBackupper) executeActions( groupResource schema.GroupResource, name, namespace string, metadata metav1.Object, + finalize bool, ) (runtime.Unstructured, error) { for _, action := range ib.backupRequest.ResolvedActions { if !action.ShouldUse(groupResource, namespace, metadata, log) { @@ -325,14 +316,49 @@ func (ib *itemBackupper) executeActions( } log.Info("Executing custom action") - // Note: we're ignoring the operationID returned from Execute for now, it will be used - // with the async plugin action implementation - updatedItem, additionalItemIdentifiers, _, err := action.Execute(obj, ib.backupRequest.Backup) + updatedItem, additionalItemIdentifiers, operationID, itemsToUpdate, err := action.Execute(obj, ib.backupRequest.Backup) if err != nil { return nil, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) } + u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()} mustInclude := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation] == "true" + // remove the annotation as it's for communication between BIA and velero server, + // we don't want the resource be restored with this annotation. + if _, ok := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation]; ok { + delete(u.GetAnnotations(), mustIncludeAdditionalItemAnnotation) + } + obj = u + if finalize { + continue + } + + // If async plugin started async operation, add it to the ItemOperations list + // ignore during finalize phase + if operationID != "" { + resourceIdentifier := velero.ResourceIdentifier{ + GroupResource: groupResource, + Namespace: namespace, + Name: name, + } + now := metav1.Now() + newOperation := itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + BackupName: ib.backupRequest.Backup.Name, + BackupUID: string(ib.backupRequest.Backup.UID), + BackupItemAction: action.Name(), + ResourceIdentifier: resourceIdentifier, + OperationID: operationID, + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &now, + }, + } + newOperation.Spec.ItemsToUpdate = itemsToUpdate + itemOperList := ib.backupRequest.GetItemOperationsList() + *itemOperList = append(*itemOperList, &newOperation) + } for _, additionalItem := range additionalItemIdentifiers { gvr, resource, err := ib.discoveryHelper.ResourceFor(additionalItem.GroupResource.WithVersion("")) @@ -363,12 +389,6 @@ func (ib *itemBackupper) executeActions( return nil, err } } - // remove the annotation as it's for communication between BIA and velero server, - // we don't want the resource be restored with this annotation. - if _, ok := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation]; ok { - delete(u.GetAnnotations(), mustIncludeAdditionalItemAnnotation) - } - obj = u } return obj, nil } diff --git a/pkg/backup/item_collector.go b/pkg/backup/item_collector.go index 88be4e0789..a176e7c22a 100644 --- a/pkg/backup/item_collector.go +++ b/pkg/backup/item_collector.go @@ -37,6 +37,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/collections" ) @@ -58,11 +59,27 @@ type kubernetesResource struct { namespace, name, path string } +// getItemsFromResourceIdentifiers converts ResourceIdentifiers to +// kubernetesResources +func (r *itemCollector) getItemsFromResourceIdentifiers(resourceIDs []velero.ResourceIdentifier) []*kubernetesResource { + + grResourceIDsMap := make(map[schema.GroupResource][]velero.ResourceIdentifier) + for _, resourceID := range resourceIDs { + grResourceIDsMap[resourceID.GroupResource] = append(grResourceIDsMap[resourceID.GroupResource], resourceID) + } + return r.getItems(grResourceIDsMap) +} + // getAllItems gets all relevant items from all API groups. func (r *itemCollector) getAllItems() []*kubernetesResource { + return r.getItems(nil) +} + +// getAllItems gets all relevant items from all API groups. +func (r *itemCollector) getItems(resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier) []*kubernetesResource { var resources []*kubernetesResource for _, group := range r.discoveryHelper.Resources() { - groupItems, err := r.getGroupItems(r.log, group) + groupItems, err := r.getGroupItems(r.log, group, resourceIDsMap) if err != nil { r.log.WithError(err).WithField("apiGroup", group.String()).Error("Error collecting resources from API group") continue @@ -75,7 +92,7 @@ func (r *itemCollector) getAllItems() []*kubernetesResource { } // getGroupItems collects all relevant items from a single API group. -func (r *itemCollector) getGroupItems(log logrus.FieldLogger, group *metav1.APIResourceList) ([]*kubernetesResource, error) { +func (r *itemCollector) getGroupItems(log logrus.FieldLogger, group *metav1.APIResourceList, resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier) ([]*kubernetesResource, error) { log = log.WithField("group", group.GroupVersion) log.Infof("Getting items for group") @@ -93,7 +110,7 @@ func (r *itemCollector) getGroupItems(log logrus.FieldLogger, group *metav1.APIR var items []*kubernetesResource for _, resource := range group.APIResources { - resourceItems, err := r.getResourceItems(log, gv, resource) + resourceItems, err := r.getResourceItems(log, gv, resource, resourceIDsMap) if err != nil { log.WithError(err).WithField("resource", resource.String()).Error("Error getting items for resource") continue @@ -164,7 +181,7 @@ func getOrderedResourcesForType(orderedResources map[string]string, resourceType } // getResourceItems collects all relevant items for a given group-version-resource. -func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.GroupVersion, resource metav1.APIResource) ([]*kubernetesResource, error) { +func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.GroupVersion, resource metav1.APIResource, resourceIDsMap map[schema.GroupResource][]velero.ResourceIdentifier) ([]*kubernetesResource, error) { log = log.WithField("resource", resource.Name) log.Info("Getting items for resource") @@ -182,6 +199,45 @@ func (r *itemCollector) getResourceItems(log logrus.FieldLogger, gv schema.Group return nil, errors.WithStack(err) } + // If we have a resourceIDs map, then only return items listed in it + if resourceIDsMap != nil { + resourceIDs, ok := resourceIDsMap[gr] + if !ok { + log.Info("Skipping resource because no items found in supplied ResourceIdentifier list") + return nil, nil + } + var items []*kubernetesResource + for _, resourceID := range resourceIDs { + log.WithFields( + logrus.Fields{ + "namespace": resourceID.Namespace, + "name": resourceID.Name, + }, + ).Infof("Getting item") + resourceClient, err := r.dynamicFactory.ClientForGroupVersionResource(gv, resource, resourceID.Namespace) + unstructured, err := resourceClient.Get(resourceID.Name, metav1.GetOptions{}) + if err != nil { + log.WithError(errors.WithStack(err)).Error("Error getting item") + continue + } + + path, err := r.writeToFile(unstructured) + if err != nil { + log.WithError(err).Error("Error writing item to file") + continue + } + + items = append(items, &kubernetesResource{ + groupResource: gr, + preferredGVR: preferredGVR, + namespace: resourceID.Namespace, + name: resourceID.Name, + path: path, + }) + } + + return items, nil + } // If the resource we are backing up is NOT namespaces, and it is cluster-scoped, check to see if // we should include it based on the IncludeClusterResources setting. if gr != kuberesource.Namespaces && clusterScoped { diff --git a/pkg/backup/request.go b/pkg/backup/request.go index a94faa2d77..24b7ecaa67 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -24,6 +24,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/volume" @@ -51,6 +52,16 @@ type Request struct { PodVolumeBackups []*velerov1api.PodVolumeBackup BackedUpItems map[itemKey]struct{} CSISnapshots []snapshotv1api.VolumeSnapshot + itemOperationsList *[]*itemoperation.BackupOperation +} + +// GetItemOperationsList returns ItemOperationsList, initializing it if necessary +func (r *Request) GetItemOperationsList() *[]*itemoperation.BackupOperation { + if r.itemOperationsList == nil { + list := []*itemoperation.BackupOperation{} + r.itemOperationsList = &list + } + return r.itemOperationsList } // BackupResourceList returns the list of backed up resources grouped by the API diff --git a/pkg/builder/backup_builder.go b/pkg/builder/backup_builder.go index f18ce49092..a599816bad 100644 --- a/pkg/builder/backup_builder.go +++ b/pkg/builder/backup_builder.go @@ -245,3 +245,9 @@ func (b *BackupBuilder) CSISnapshotTimeout(timeout time.Duration) *BackupBuilder b.object.Spec.CSISnapshotTimeout.Duration = timeout return b } + +// ItemOperationTimeout sets the Backup's ItemOperationTimeout +func (b *BackupBuilder) ItemOperationTimeout(timeout time.Duration) *BackupBuilder { + b.object.Spec.ItemOperationTimeout.Duration = timeout + return b +} diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index 52aeab3d52..ee16a08524 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -99,6 +99,7 @@ type CreateOptions struct { FromSchedule string OrderedResources string CSISnapshotTimeout time.Duration + ItemOperationTimeout time.Duration client veleroclient.Interface } @@ -124,6 +125,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.VarP(&o.Selector, "selector", "l", "Only back up resources matching this label selector.") flags.StringVar(&o.OrderedResources, "ordered-resources", "", "Mapping Kinds to an ordered list of specific resources of that Kind. Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon. Example: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'. Optional.") flags.DurationVar(&o.CSISnapshotTimeout, "csi-snapshot-timeout", o.CSISnapshotTimeout, "How long to wait for CSI snapshot creation before timeout.") + flags.DurationVar(&o.ItemOperationTimeout, "item-operation-timeout", o.ItemOperationTimeout, "How long to wait for async plugin operations before timeout.") f := flags.VarPF(&o.SnapshotVolumes, "snapshot-volumes", "", "Take snapshots of PersistentVolumes as part of the backup. If the parameter is not set, it is treated as setting to 'true'.") // this allows the user to just specify "--snapshot-volumes" as shorthand for "--snapshot-volumes=true" // like a normal bool flag @@ -335,7 +337,8 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro TTL(o.TTL). StorageLocation(o.StorageLocation). VolumeSnapshotLocations(o.SnapshotLocations...). - CSISnapshotTimeout(o.CSISnapshotTimeout) + CSISnapshotTimeout(o.CSISnapshotTimeout). + ItemOperationTimeout(o.ItemOperationTimeout) if len(o.OrderedResources) > 0 { orders, err := ParseOrderedResources(o.OrderedResources) if err != nil { diff --git a/pkg/cmd/cli/backup/create_test.go b/pkg/cmd/cli/backup/create_test.go index 8941630256..5e7b0161b6 100644 --- a/pkg/cmd/cli/backup/create_test.go +++ b/pkg/cmd/cli/backup/create_test.go @@ -37,6 +37,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) { o.OrderedResources = "pods=p1,p2;persistentvolumeclaims=pvc1,pvc2" orders, err := ParseOrderedResources(o.OrderedResources) o.CSISnapshotTimeout = 20 * time.Minute + o.ItemOperationTimeout = 20 * time.Minute assert.NoError(t, err) backup, err := o.BuildBackup(testNamespace) @@ -49,6 +50,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) { IncludeClusterResources: o.IncludeClusterResources.Value, OrderedResources: orders, CSISnapshotTimeout: metav1.Duration{Duration: o.CSISnapshotTimeout}, + ItemOperationTimeout: metav1.Duration{Duration: o.ItemOperationTimeout}, }, backup.Spec) assert.Equal(t, map[string]string{ diff --git a/pkg/cmd/cli/backup/logs.go b/pkg/cmd/cli/backup/logs.go index 554fa17137..95c2387a31 100644 --- a/pkg/cmd/cli/backup/logs.go +++ b/pkg/cmd/cli/backup/logs.go @@ -63,8 +63,8 @@ func NewLogsCommand(f client.Factory) *cobra.Command { } switch backup.Status.Phase { - case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed: - // terminal phases, do nothing. + case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: + // terminal and waiting for plugin operations phases, do nothing. default: cmd.Exit("Logs for backup %q are not available until it's finished processing. Please wait "+ "until the backup has a phase of Completed or Failed and try again.", backupName) diff --git a/pkg/cmd/cli/schedule/create.go b/pkg/cmd/cli/schedule/create.go index bcccd53739..8aac85b8a5 100644 --- a/pkg/cmd/cli/schedule/create.go +++ b/pkg/cmd/cli/schedule/create.go @@ -146,6 +146,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { DefaultVolumesToFsBackup: o.BackupOptions.DefaultVolumesToFsBackup.Value, OrderedResources: orders, CSISnapshotTimeout: metav1.Duration{Duration: o.BackupOptions.CSISnapshotTimeout}, + ItemOperationTimeout: metav1.Duration{Duration: o.BackupOptions.ItemOperationTimeout}, }, Schedule: o.Schedule, UseOwnerReferencesInBackup: &o.UseOwnerReferencesInBackup, diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 377646a758..880e29f1e6 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -102,7 +102,8 @@ const ( // the default TTL for a backup defaultBackupTTL = 30 * 24 * time.Hour - defaultCSISnapshotTimeout = 10 * time.Minute + defaultCSISnapshotTimeout = 10 * time.Minute + defaultItemOperationTimeout = 60 * time.Minute // defaultCredentialsDirectory is the path on disk where credential // files will be written to @@ -114,6 +115,7 @@ type serverConfig struct { pluginDir, metricsAddress, defaultBackupLocation string backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout time.Duration defaultBackupTTL, storeValidationFrequency, defaultCSISnapshotTimeout time.Duration + defaultItemOperationTimeout time.Duration restoreResourcePriorities restore.Priorities defaultVolumeSnapshotLocations map[string]string restoreOnly bool @@ -125,6 +127,7 @@ type serverConfig struct { formatFlag *logging.FormatFlag repoMaintenanceFrequency time.Duration garbageCollectionFrequency time.Duration + itemOperationSyncFrequency time.Duration defaultVolumesToFsBackup bool uploaderType string } @@ -146,6 +149,7 @@ func NewCommand(f client.Factory) *cobra.Command { backupSyncPeriod: defaultBackupSyncPeriod, defaultBackupTTL: defaultBackupTTL, defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, + defaultItemOperationTimeout: defaultItemOperationTimeout, storeValidationFrequency: defaultStoreValidationFrequency, podVolumeOperationTimeout: defaultPodVolumeOperationTimeout, restoreResourcePriorities: defaultRestorePriorities, @@ -221,8 +225,10 @@ func NewCommand(f client.Factory) *cobra.Command { command.Flags().DurationVar(&config.defaultBackupTTL, "default-backup-ttl", config.defaultBackupTTL, "How long to wait by default before backups can be garbage collected.") command.Flags().DurationVar(&config.repoMaintenanceFrequency, "default-repo-maintain-frequency", config.repoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default.") command.Flags().DurationVar(&config.garbageCollectionFrequency, "garbage-collection-frequency", config.garbageCollectionFrequency, "How often garbage collection is run for expired backups.") + command.Flags().DurationVar(&config.itemOperationSyncFrequency, "item-operation-sync-frequency", config.itemOperationSyncFrequency, "How often to check status on async backup/restore operations after backup processing.") command.Flags().BoolVar(&config.defaultVolumesToFsBackup, "default-volumes-to-fs-backup", config.defaultVolumesToFsBackup, "Backup all volumes with pod volume file system backup by default.") command.Flags().StringVar(&config.uploaderType, "uploader-type", config.uploaderType, "Type of uploader to handle the transfer of data of pod volumes") + command.Flags().DurationVar(&config.defaultItemOperationTimeout, "default-item-operation-timeout", config.defaultItemOperationTimeout, "How long to wait on asynchronous BackupItemActions and RestoreItemActions to complete before timing out.") return command } @@ -649,6 +655,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.config.defaultVolumesToFsBackup, s.config.defaultBackupTTL, s.config.defaultCSISnapshotTimeout, + s.config.defaultItemOperationTimeout, s.sharedInformerFactory.Velero().V1().VolumeSnapshotLocations().Lister(), defaultVolumeSnapshotLocations, s.metrics, @@ -674,14 +681,16 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } // Note: all runtime type controllers that can be disabled are grouped separately, below: enabledRuntimeControllers := map[string]struct{}{ - controller.ServerStatusRequest: {}, - controller.DownloadRequest: {}, - controller.Schedule: {}, - controller.BackupRepo: {}, - controller.BackupDeletion: {}, - controller.GarbageCollection: {}, - controller.BackupSync: {}, - controller.Restore: {}, + controller.ServerStatusRequest: {}, + controller.DownloadRequest: {}, + controller.Schedule: {}, + controller.BackupRepo: {}, + controller.BackupDeletion: {}, + controller.BackupFinalizer: {}, + controller.GarbageCollection: {}, + controller.BackupSync: {}, + controller.AsyncBackupOperations: {}, + controller.Restore: {}, } if s.config.restoreOnly { @@ -691,6 +700,8 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string controller.Schedule, controller.GarbageCollection, controller.BackupDeletion, + controller.BackupFinalizer, + controller.AsyncBackupOperations, ) } @@ -785,6 +796,51 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } } + var backupOpsMap *controller.BackupItemOperationsMap + if _, ok := enabledRuntimeControllers[controller.AsyncBackupOperations]; ok { + r, m := controller.NewAsyncBackupOperationsReconciler( + s.logger, + s.mgr.GetClient(), + s.config.itemOperationSyncFrequency, + newPluginManager, + backupStoreGetter, + s.metrics, + ) + if err := r.SetupWithManager(s.mgr); err != nil { + s.logger.Fatal(err, "unable to create controller", "controller", controller.AsyncBackupOperations) + } + backupOpsMap = m + } + + if _, ok := enabledRuntimeControllers[controller.BackupFinalizer]; ok { + backupper, err := backup.NewKubernetesBackupper( + s.veleroClient.VeleroV1(), + s.discoveryHelper, + client.NewDynamicFactory(s.dynamicClient), + podexec.NewPodCommandExecutor(s.kubeClientConfig, s.kubeClient.CoreV1().RESTClient()), + podvolume.NewBackupperFactory(s.repoLocker, s.repoEnsurer, s.veleroClient, s.kubeClient.CoreV1(), + s.kubeClient.CoreV1(), s.kubeClient.CoreV1(), + s.sharedInformerFactory.Velero().V1().BackupRepositories().Informer().HasSynced, s.logger), + s.config.podVolumeOperationTimeout, + s.config.defaultVolumesToFsBackup, + s.config.clientPageSize, + s.config.uploaderType, + ) + cmd.CheckError(err) + r := controller.NewBackupFinalizerReconciler( + s.mgr.GetClient(), + clock.RealClock{}, + backupper, + newPluginManager, + backupStoreGetter, + s.logger, + s.metrics, + ) + if err := r.SetupWithManager(s.mgr); err != nil { + s.logger.Fatal(err, "unable to create controller", "controller", controller.BackupFinalizer) + } + } + if _, ok := enabledRuntimeControllers[controller.DownloadRequest]; ok { r := controller.NewDownloadRequestReconciler( s.mgr.GetClient(), @@ -792,6 +848,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string newPluginManager, backupStoreGetter, s.logger, + backupOpsMap, ) if err := r.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", controller.DownloadRequest) diff --git a/pkg/cmd/server/server_test.go b/pkg/cmd/server/server_test.go index ddf9fb9d37..d25066d65f 100644 --- a/pkg/cmd/server/server_test.go +++ b/pkg/cmd/server/server_test.go @@ -89,6 +89,7 @@ func TestRemoveControllers(t *testing.T) { { name: "Remove all disable controllers", disabledControllers: []string{ + controller.AsyncBackupOperations, controller.Backup, controller.BackupDeletion, controller.BackupSync, @@ -127,11 +128,12 @@ func TestRemoveControllers(t *testing.T) { } enabledRuntimeControllers := map[string]struct{}{ - controller.ServerStatusRequest: {}, - controller.Schedule: {}, - controller.BackupDeletion: {}, - controller.BackupRepo: {}, - controller.DownloadRequest: {}, + controller.ServerStatusRequest: {}, + controller.Schedule: {}, + controller.BackupDeletion: {}, + controller.BackupRepo: {}, + controller.DownloadRequest: {}, + controller.AsyncBackupOperations: {}, } totalNumOriginalControllers := len(enabledControllers) + len(enabledRuntimeControllers) diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 9be8ed59ee..8c73e6cf7f 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -35,6 +35,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/features" clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/volume" ) @@ -66,6 +67,8 @@ func DescribeBackup( case velerov1api.BackupPhaseCompleted: phaseString = color.GreenString(phaseString) case velerov1api.BackupPhaseDeleting: + case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: + case velerov1api.BackupPhaseFinalizingAfterPluginOperations, velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed: case velerov1api.BackupPhaseInProgress: case velerov1api.BackupPhaseNew: } @@ -166,6 +169,7 @@ func DescribeBackupSpec(d *Describer, spec velerov1api.BackupSpec) { d.Println() d.Printf("CSISnapshotTimeout:\t%s\n", spec.CSISnapshotTimeout.Duration) + d.Printf("ItemOperationTimeout:\t%s\n", spec.ItemOperationTimeout.Duration) d.Println() if len(spec.Hooks.Resources) == 0 { @@ -284,6 +288,8 @@ func DescribeBackupStatus(ctx context.Context, kbClient kbclient.Client, d *Desc d.Println() } + describeAsyncBackupItemOperations(ctx, kbClient, d, backup, details, insecureSkipTLSVerify, caCertPath) + if details { describeBackupResourceList(ctx, kbClient, d, backup, insecureSkipTLSVerify, caCertPath) d.Println() @@ -317,6 +323,33 @@ func DescribeBackupStatus(ctx context.Context, kbClient kbclient.Client, d *Desc d.Printf("Velero-Native Snapshots: \n") } +func describeAsyncBackupItemOperations(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string) { + status := backup.Status + if status.AsyncBackupItemOperationsAttempted > 0 { + if !details { + d.Printf("Async Backup Item Operations:\t%d of %d completed successfully, %d failed (specify --details for more information)\n", status.AsyncBackupItemOperationsCompleted, status.AsyncBackupItemOperationsAttempted, status.AsyncBackupItemOperationsFailed) + return + } + + buf := new(bytes.Buffer) + if err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + d.Printf("Async Backup Item Operations:\t\n", err) + return + } + + var operations []*itemoperation.BackupOperation + if err := json.NewDecoder(buf).Decode(&operations); err != nil { + d.Printf("Async Backup Item Operations:\t\n", err) + return + } + + d.Printf("Async Backup Item Operations:\n") + for _, operation := range operations { + describeAsyncBackupItemOperation(d, operation) + } + } +} + func describeBackupResourceList(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) { buf := new(bytes.Buffer) if err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupResourceList, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { @@ -365,6 +398,40 @@ func describeSnapshot(d *Describer, pvName, snapshotID, volumeType, volumeAZ str d.Printf("\t\tIOPS:\t%s\n", iopsString) } +func describeAsyncBackupItemOperation(d *Describer, operation *itemoperation.BackupOperation) { + d.Printf("\tOperation for %s %s/%s:\n", operation.Spec.ResourceIdentifier, operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) + d.Printf("\t\tBackup Item Action Plugin:\t%s\n", operation.Spec.BackupItemAction) + d.Printf("\t\tOperation ID:\t%s\n", operation.Spec.OperationID) + if len(operation.Spec.ItemsToUpdate) > 0 { + d.Printf("\t\tItems to Update:\n") + } + for _, item := range operation.Spec.ItemsToUpdate { + d.Printf("\t\t\t%s %s/%s\n", item, item.Namespace, item.Name) + } + d.Printf("\t\tPhase:\t%s\n", operation.Status.Phase) + if operation.Status.Error != "" { + d.Printf("\t\tOperation Error:\t%s\n", operation.Status.Error) + } + if operation.Status.NTotal > 0 || operation.Status.NCompleted > 0 { + d.Printf("\t\tProgress:\t%v of %v complete (%s)\n", + operation.Status.NCompleted, + operation.Status.NTotal, + operation.Status.OperationUnits) + } + if operation.Status.Description != "" { + d.Printf("\t\tProgress description:\t%s\n", operation.Status.Description) + } + if operation.Status.Created != nil { + d.Printf("\t\tCreated:\t%s\n", operation.Status.Created.String()) + } + if operation.Status.Started != nil { + d.Printf("\t\tStarted:\t%s\n", operation.Status.Started.String()) + } + if operation.Status.Updated != nil { + d.Printf("\t\tUpdated:\t%s\n", operation.Status.Updated.String()) + } +} + // DescribeDeleteBackupRequests describes delete backup requests in human-readable format. func DescribeDeleteBackupRequests(d *Describer, requests []velerov1api.DeleteBackupRequest) { d.Printf("Deletion Attempts") diff --git a/pkg/controller/async_backup_operations_controller.go b/pkg/controller/async_backup_operations_controller.go new file mode 100644 index 0000000000..e945a2ad44 --- /dev/null +++ b/pkg/controller/async_backup_operations_controller.go @@ -0,0 +1,483 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clocks "k8s.io/utils/clock" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/metrics" + "github.com/vmware-tanzu/velero/pkg/persistence" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/util/encode" + "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +const ( + defaultAsyncBackupOperationsFrequency = 2 * time.Minute +) + +type operationsForBackup struct { + operations []*itemoperation.BackupOperation + changesSinceUpdate bool + errsSinceUpdate []string +} + +// FIXME: remove if handled by backup finalizer controller +func (o *operationsForBackup) anyItemsToUpdate() bool { + for _, op := range o.operations { + if len(op.Spec.ItemsToUpdate) > 0 { + return true + } + } + return false +} +func (in *operationsForBackup) DeepCopy() *operationsForBackup { + if in == nil { + return nil + } + out := new(operationsForBackup) + in.DeepCopyInto(out) + return out +} + +func (in *operationsForBackup) DeepCopyInto(out *operationsForBackup) { + *out = *in + if in.operations != nil { + in, out := &in.operations, &out.operations + *out = make([]*itemoperation.BackupOperation, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(itemoperation.BackupOperation) + (*in).DeepCopyInto(*out) + } + } + } + if in.errsSinceUpdate != nil { + in, out := &in.errsSinceUpdate, &out.errsSinceUpdate + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +func (o *operationsForBackup) uploadProgress(backupStore persistence.BackupStore, backupName string) error { + if len(o.operations) > 0 { + var backupItemOperations *bytes.Buffer + backupItemOperations, errs := encodeToJSONGzip(o.operations, "backup item operations list") + if errs != nil { + return errors.Wrap(errs[0], "error encoding item operations json") + } + err := backupStore.PutBackupItemOperations(backupName, backupItemOperations) + if err != nil { + return errors.Wrap(err, "error uploading item operations json") + } + } + o.changesSinceUpdate = false + o.errsSinceUpdate = nil + return nil +} + +type BackupItemOperationsMap struct { + operations map[string]*operationsForBackup + opsLock sync.Mutex +} + +// If backup has changes not yet uploaded, upload them now +func (m *BackupItemOperationsMap) UpdateForBackup(backupStore persistence.BackupStore, backupName string) error { + // lock operations map + m.opsLock.Lock() + defer m.opsLock.Unlock() + + operations, ok := m.operations[backupName] + // if operations for this backup aren't found, or if there are no changes + // or errors since last update, do nothing + if !ok || (!operations.changesSinceUpdate && len(operations.errsSinceUpdate) == 0) { + return nil + } + if err := operations.uploadProgress(backupStore, backupName); err != nil { + return err + } + return nil +} + +type asyncBackupOperationsReconciler struct { + client.Client + logger logrus.FieldLogger + clock clocks.WithTickerAndDelayedExecution + frequency time.Duration + itemOperationsMap *BackupItemOperationsMap + newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager + backupStoreGetter persistence.ObjectBackupStoreGetter + metrics *metrics.ServerMetrics +} + +func NewAsyncBackupOperationsReconciler( + logger logrus.FieldLogger, + client client.Client, + frequency time.Duration, + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, + backupStoreGetter persistence.ObjectBackupStoreGetter, + metrics *metrics.ServerMetrics, +) (*asyncBackupOperationsReconciler, *BackupItemOperationsMap) { + abor := &asyncBackupOperationsReconciler{ + Client: client, + logger: logger, + clock: clocks.RealClock{}, + frequency: frequency, + itemOperationsMap: &BackupItemOperationsMap{operations: make(map[string]*operationsForBackup)}, + newPluginManager: newPluginManager, + backupStoreGetter: backupStoreGetter, + metrics: metrics, + } + if abor.frequency <= 0 { + abor.frequency = defaultAsyncBackupOperationsFrequency + } + return abor, abor.itemOperationsMap +} + +func (c *asyncBackupOperationsReconciler) SetupWithManager(mgr ctrl.Manager) error { + s := kube.NewPeriodicalEnqueueSource(c.logger, mgr.GetClient(), &velerov1api.BackupList{}, c.frequency, kube.PeriodicalEnqueueSourceOption{}) + return ctrl.NewControllerManagedBy(mgr). + For(&velerov1api.Backup{}, builder.WithPredicates(kube.FalsePredicate{})). + Watches(s, nil). + Complete(c) +} + +// +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get +// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get +func (c *asyncBackupOperationsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := c.logger.WithField("async backup operations for backup", req.String()) + // FIXME: make this log.Debug + log.Info("asyncBackupOperationsReconciler getting backup") + + original := &velerov1api.Backup{} + if err := c.Get(ctx, req.NamespacedName, original); err != nil { + if apierrors.IsNotFound(err) { + log.WithError(err).Error("backup not found") + return ctrl.Result{}, nil + } + return ctrl.Result{}, errors.Wrapf(err, "error getting backup %s", req.String()) + } + backup := original.DeepCopy() + log.Debugf("backup: %s", backup.Name) + + log = c.logger.WithFields( + logrus.Fields{ + "backup": req.String(), + }, + ) + + switch backup.Status.Phase { + case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: + // only process backups waiting for plugin operations to complete + default: + log.Debug("Backup has no ongoing async plugin operations, skipping") + return ctrl.Result{}, nil + } + + loc := &velerov1api.BackupStorageLocation{} + if err := c.Get(ctx, client.ObjectKey{ + Namespace: req.Namespace, + Name: backup.Spec.StorageLocation, + }, loc); err != nil { + if apierrors.IsNotFound(err) { + log.Warnf("Cannot check progress on async Backup operations because backup storage location %s does not exist; marking backup PartiallyFailed", backup.Spec.StorageLocation) + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } else { + log.Warnf("Cannot check progress on async Backup operations because backup storage location %s could not be retrieved: %s; marking backup PartiallyFailed", backup.Spec.StorageLocation, err.Error()) + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } + err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &operationsForBackup{errsSinceUpdate: []string{err.Error()}}, false, false) + if err2 != nil { + log.WithError(err2).Error("error updating Backup") + } + return ctrl.Result{}, errors.Wrap(err, "error getting backup storage location") + } + + if loc.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { + log.Infof("Cannot check progress on async Backup operations because backup storage location %s is currently in read-only mode; marking backup PartiallyFailed", loc.Name) + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + + err := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &operationsForBackup{errsSinceUpdate: []string{"BSL is read-only"}}, false, false) + if err != nil { + log.WithError(err).Error("error updating Backup") + } + return ctrl.Result{}, nil + } + + pluginManager := c.newPluginManager(c.logger) + defer pluginManager.CleanupClients() + backupStore, err := c.backupStoreGetter.Get(loc, pluginManager, c.logger) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "error getting backup store") + } + + operations, err := c.getOperationsForBackup(backupStore, backup.Name) + if err != nil { + err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, &operationsForBackup{errsSinceUpdate: []string{err.Error()}}, false, false) + if err2 != nil { + return ctrl.Result{}, errors.Wrap(err2, "error updating Backup") + } + return ctrl.Result{}, errors.Wrap(err, "error getting backup operations") + } + stillInProgress, changes, opsCompleted, opsFailed, errs := getBackupItemOperationProgress(backup, pluginManager, operations.operations) + // if len(errs)>0, need to update backup errors and error log + operations.errsSinceUpdate = append(operations.errsSinceUpdate, errs...) + backup.Status.Errors += len(operations.errsSinceUpdate) + asyncCompletionChanges := false + if backup.Status.AsyncBackupItemOperationsCompleted != opsCompleted || backup.Status.AsyncBackupItemOperationsFailed != opsFailed { + asyncCompletionChanges = true + backup.Status.AsyncBackupItemOperationsCompleted = opsCompleted + backup.Status.AsyncBackupItemOperationsFailed = opsFailed + } + if changes { + operations.changesSinceUpdate = true + } + + // if stillInProgress is false, backup moves to finalize phase and needs update + // if operations.errsSinceUpdate is not empty, then backup phase needs to change to + // BackupPhaseWaitingForPluginOperationsPartiallyFailed and needs update + // If the only changes are incremental progress, then no write is necessary, progress can remain in memory + if !stillInProgress { + if len(operations.errsSinceUpdate) > 0 { + backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed + } + if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations { + log.Infof("Marking backup %s FinalizingAfterPluginOperations", backup.Name) + backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperations + } else { + log.Infof("Marking backup %s FinalizingAfterPluginOperationsPartiallyFailed", backup.Name) + backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed + } + } + err = c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, operations, asyncCompletionChanges, changes) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "error updating Backup") + } + return ctrl.Result{}, nil +} + +func (c *asyncBackupOperationsReconciler) updateBackupAndOperationsJSON( + ctx context.Context, + original, backup *velerov1api.Backup, + backupStore persistence.BackupStore, + operations *operationsForBackup, + changes bool, + asyncCompletionChanges bool) error { + + backupScheduleName := backup.GetLabels()[velerov1api.ScheduleNameLabel] + + if len(operations.errsSinceUpdate) > 0 { + c.metrics.RegisterBackupItemsErrorsGauge(backupScheduleName, backup.Status.Errors) + // FIXME: download/upload results once https://github.com/vmware-tanzu/velero/pull/5576 is merged + } + removeIfComplete := true + defer func() { + // remove local operations list if complete + c.itemOperationsMap.opsLock.Lock() + if removeIfComplete && (backup.Status.Phase == velerov1api.BackupPhaseCompleted || + backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed) { + + c.deleteOperationsForBackup(backup.Name) + } else if changes { + c.putOperationsForBackup(operations, backup.Name) + } + c.itemOperationsMap.opsLock.Unlock() + }() + + // update backup and upload progress if errs or complete + if len(operations.errsSinceUpdate) > 0 || + backup.Status.Phase == velerov1api.BackupPhaseCompleted || + backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed { + // update file store + if backupStore != nil { + backupJSON := new(bytes.Buffer) + if err := encode.EncodeTo(backup, "json", backupJSON); err != nil { + removeIfComplete = false + return errors.Wrap(err, "error encoding backup json") + } + err := backupStore.PutBackupMetadata(backup.Name, backupJSON) + if err != nil { + removeIfComplete = false + return errors.Wrap(err, "error uploading backup json") + } + if err := operations.uploadProgress(backupStore, backup.Name); err != nil { + removeIfComplete = false + return err + } + } + // update backup + err := c.Client.Patch(ctx, backup, client.MergeFrom(original)) + if err != nil { + removeIfComplete = false + return errors.Wrapf(err, "error updating Backup %s", backup.Name) + } + } else if asyncCompletionChanges { + // If backup is still incomplete and no new errors are found but there are some new operations + // completed, patch backup to reflect new completion numbers, but don't upload detailed json file + err := c.Client.Patch(ctx, backup, client.MergeFrom(original)) + if err != nil { + return errors.Wrapf(err, "error updating Backup %s", backup.Name) + } + } + return nil +} + +// returns a deep copy so we can minimize the time the map is locked +func (c *asyncBackupOperationsReconciler) getOperationsForBackup( + backupStore persistence.BackupStore, + backupName string) (*operationsForBackup, error) { + var err error + // lock operations map + c.itemOperationsMap.opsLock.Lock() + defer c.itemOperationsMap.opsLock.Unlock() + + operations, ok := c.itemOperationsMap.operations[backupName] + if !ok || len(operations.operations) == 0 { + operations = &operationsForBackup{} + operations.operations, err = backupStore.GetBackupItemOperations(backupName) + if err == nil { + c.itemOperationsMap.operations[backupName] = operations + } + } + return operations.DeepCopy(), err +} + +func (c *asyncBackupOperationsReconciler) putOperationsForBackup( + operations *operationsForBackup, + backupName string) { + if operations != nil { + c.itemOperationsMap.operations[backupName] = operations + } +} + +func (c *asyncBackupOperationsReconciler) deleteOperationsForBackup(backupName string) { + if _, ok := c.itemOperationsMap.operations[backupName]; ok { + delete(c.itemOperationsMap.operations, backupName) + } + return +} + +func getBackupItemOperationProgress( + backup *velerov1api.Backup, + pluginManager clientmgmt.Manager, + operationsList []*itemoperation.BackupOperation) (bool, bool, int, int, []string) { + inProgressOperations := false + changes := false + var errs []string + var completedCount, failedCount int + + for _, operation := range operationsList { + if operation.Status.Phase == itemoperation.OperationPhaseInProgress { + bia, err := pluginManager.GetBackupItemActionV2(operation.Spec.BackupItemAction) + if err != nil { + operation.Status.Phase = itemoperation.OperationPhaseFailed + operation.Status.Error = err.Error() + errs = append(errs, err.Error()) + changes = true + failedCount++ + continue + } + operationProgress, err := bia.Progress(operation.Spec.OperationID, backup) + if err != nil { + operation.Status.Phase = itemoperation.OperationPhaseFailed + operation.Status.Error = err.Error() + errs = append(errs, err.Error()) + changes = true + failedCount++ + continue + } + if operation.Status.NCompleted != operationProgress.NCompleted { + operation.Status.NCompleted = operationProgress.NCompleted + changes = true + } + if operation.Status.NTotal != operationProgress.NTotal { + operation.Status.NTotal = operationProgress.NTotal + changes = true + } + if operation.Status.OperationUnits != operationProgress.OperationUnits { + operation.Status.OperationUnits = operationProgress.OperationUnits + changes = true + } + if operation.Status.Description != operationProgress.Description { + operation.Status.Description = operationProgress.Description + changes = true + } + started := metav1.NewTime(operationProgress.Started) + if operation.Status.Started == nil || *(operation.Status.Started) != started { + operation.Status.Started = &started + changes = true + } + updated := metav1.NewTime(operationProgress.Updated) + if operation.Status.Updated == nil || *(operation.Status.Updated) != updated { + operation.Status.Updated = &updated + changes = true + } + + if operationProgress.Completed { + if operationProgress.Err != "" { + operation.Status.Phase = itemoperation.OperationPhaseFailed + operation.Status.Error = operationProgress.Err + errs = append(errs, operationProgress.Err) + changes = true + failedCount++ + continue + } + operation.Status.Phase = itemoperation.OperationPhaseCompleted + changes = true + completedCount++ + continue + } + // cancel operation if past timeout period + if operation.Status.Created.Time.Add(backup.Spec.ItemOperationTimeout.Duration).Before(time.Now()) { + _ = bia.Cancel(operation.Spec.OperationID, backup) + operation.Status.Phase = itemoperation.OperationPhaseFailed + operation.Status.Error = "Asynchronous action timed out" + errs = append(errs, operation.Status.Error) + changes = true + failedCount++ + continue + } + // if we reach this point, the operation is still running + inProgressOperations = true + } else if operation.Status.Phase == itemoperation.OperationPhaseCompleted { + completedCount++ + } else if operation.Status.Phase == itemoperation.OperationPhaseFailed { + failedCount++ + } + } + return inProgressOperations, changes, completedCount, failedCount, errs +} diff --git a/pkg/controller/async_backup_operations_controller_test.go b/pkg/controller/async_backup_operations_controller_test.go new file mode 100644 index 0000000000..935f92ac47 --- /dev/null +++ b/pkg/controller/async_backup_operations_controller_test.go @@ -0,0 +1,308 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + testclocks "k8s.io/utils/clock/testing" + ctrl "sigs.k8s.io/controller-runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/metrics" + persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/backupitemaction/v2" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +var ( + pluginManager = &pluginmocks.Manager{} + backupStore = &persistencemocks.BackupStore{} + bia = &biav2mocks.BackupItemAction{} +) + +func mockAsyncBackupOperationsReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeClock, freq time.Duration) (*asyncBackupOperationsReconciler, *BackupItemOperationsMap) { + abor, biaMap := NewAsyncBackupOperationsReconciler( + logrus.StandardLogger(), + fakeClient, + freq, + func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewFakeSingleObjectBackupStoreGetter(backupStore), + metrics.NewServerMetrics(), + ) + abor.clock = fakeClock + return abor, biaMap +} + +func TestAsyncBackupOperationsReconcile(t *testing.T) { + fakeClock := testclocks.NewFakeClock(time.Now()) + metav1Now := metav1.NewTime(fakeClock.Now()) + + defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Result() + + tests := []struct { + name string + backup *velerov1api.Backup + backupOperations []*itemoperation.BackupOperation + backupLocation *velerov1api.BackupStorageLocation + operationComplete bool + operationErr string + expectError bool + expectPhase velerov1api.BackupPhase + }{ + { + name: "WaitingForPluginOperations backup with completed operations is FinalizingAfterPluginOperations", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + backupLocation: defaultBackupLocation, + operationComplete: true, + expectPhase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-1", + BackupUID: "foo", + BackupItemAction: "foo", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-1", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperations backup with incomplete operations is still incomplete", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + backupLocation: defaultBackupLocation, + operationComplete: false, + expectPhase: velerov1api.BackupPhaseWaitingForPluginOperations, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-2", + BackupUID: "foo-2", + BackupItemAction: "foo-2", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-2", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperations backup with completed failed operations is FinalizingAfterPluginOperationsPartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-3"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + backupLocation: defaultBackupLocation, + operationComplete: true, + operationErr: "failed", + expectPhase: velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-3", + BackupUID: "foo-3", + BackupItemAction: "foo-3", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-3", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperationsPartiallyFailed backup with completed operations is FinalizingAfterPluginOperationsPartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), + backupLocation: defaultBackupLocation, + operationComplete: true, + expectPhase: velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-4", + BackupUID: "foo-4", + BackupItemAction: "foo-4", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-4", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperationsPartiallyFailed backup with incomplete operations is still incomplete", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), + backupLocation: defaultBackupLocation, + operationComplete: false, + expectPhase: velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-5", + BackupUID: "foo-5", + BackupItemAction: "foo-5", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-5", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperationsPartiallyFailed backup with completed failed operations is FinalizingAfterPluginOperationsPartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-3"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), + backupLocation: defaultBackupLocation, + operationComplete: true, + operationErr: "failed", + expectPhase: velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-6", + BackupUID: "foo-6", + BackupItemAction: "foo-6", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-6", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.backup == nil { + return + } + + initObjs := []runtime.Object{} + initObjs = append(initObjs, test.backup) + + if test.backupLocation != nil { + initObjs = append(initObjs, test.backupLocation) + } + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) + reconciler, _ := mockAsyncBackupOperationsReconciler(fakeClient, fakeClock, defaultAsyncBackupOperationsFrequency) + pluginManager.On("CleanupClients").Return(nil) + backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil) + backupStore.On("PutBackupItemOperations", mock.Anything, mock.Anything).Return(nil) + backupStore.On("PutBackupMetadata", mock.Anything, mock.Anything).Return(nil) + for _, operation := range test.backupOperations { + bia.On("Progress", operation.Spec.OperationID, mock.Anything). + Return(velero.OperationProgress{ + Completed: test.operationComplete, + Err: test.operationErr, + }, nil) + pluginManager.On("GetBackupItemActionV2", operation.Spec.BackupItemAction).Return(bia, nil) + } + _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) + gotErr := err != nil + assert.Equal(t, test.expectError, gotErr) + + backupAfter := velerov1api.Backup{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{ + Namespace: test.backup.Namespace, + Name: test.backup.Name, + }, &backupAfter) + + require.NoError(t, err) + assert.Equal(t, test.expectPhase, backupAfter.Status.Phase) + }) + } +} diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 23635704b3..cc0e034521 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -74,27 +74,28 @@ import ( type backupController struct { *genericController - discoveryHelper discovery.Helper - backupper pkgbackup.Backupper - lister velerov1listers.BackupLister - client velerov1client.BackupsGetter - kbClient kbclient.Client - clock clocks.WithTickerAndDelayedExecution - backupLogLevel logrus.Level - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - backupTracker BackupTracker - defaultBackupLocation string - defaultVolumesToFsBackup bool - defaultBackupTTL time.Duration - defaultCSISnapshotTimeout time.Duration - snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister - defaultSnapshotLocations map[string]string - metrics *metrics.ServerMetrics - backupStoreGetter persistence.ObjectBackupStoreGetter - formatFlag logging.Format - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister - volumeSnapshotClient snapshotterClientSet.Interface - credentialFileStore credentials.FileStore + discoveryHelper discovery.Helper + backupper pkgbackup.Backupper + lister velerov1listers.BackupLister + client velerov1client.BackupsGetter + kbClient kbclient.Client + clock clocks.WithTickerAndDelayedExecution + backupLogLevel logrus.Level + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + backupTracker BackupTracker + defaultBackupLocation string + defaultVolumesToFsBackup bool + defaultBackupTTL time.Duration + defaultCSISnapshotTimeout time.Duration + defaultItemOperationTimeout time.Duration + snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister + defaultSnapshotLocations map[string]string + metrics *metrics.ServerMetrics + backupStoreGetter persistence.ObjectBackupStoreGetter + formatFlag logging.Format + volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister + volumeSnapshotClient snapshotterClientSet.Interface + credentialFileStore credentials.FileStore } func NewBackupController( @@ -111,6 +112,7 @@ func NewBackupController( defaultVolumesToFsBackup bool, defaultBackupTTL time.Duration, defaultCSISnapshotTimeout time.Duration, + defaultItemOperationTimeout time.Duration, volumeSnapshotLocationLister velerov1listers.VolumeSnapshotLocationLister, defaultSnapshotLocations map[string]string, metrics *metrics.ServerMetrics, @@ -121,28 +123,29 @@ func NewBackupController( credentialStore credentials.FileStore, ) Interface { c := &backupController{ - genericController: newGenericController(Backup, logger), - discoveryHelper: discoveryHelper, - backupper: backupper, - lister: backupInformer.Lister(), - client: client, - clock: &clocks.RealClock{}, - backupLogLevel: backupLogLevel, - newPluginManager: newPluginManager, - backupTracker: backupTracker, - kbClient: kbClient, - defaultBackupLocation: defaultBackupLocation, - defaultVolumesToFsBackup: defaultVolumesToFsBackup, - defaultBackupTTL: defaultBackupTTL, - defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, - snapshotLocationLister: volumeSnapshotLocationLister, - defaultSnapshotLocations: defaultSnapshotLocations, - metrics: metrics, - backupStoreGetter: backupStoreGetter, - formatFlag: formatFlag, - volumeSnapshotLister: volumeSnapshotLister, - volumeSnapshotClient: volumeSnapshotClient, - credentialFileStore: credentialStore, + genericController: newGenericController(Backup, logger), + discoveryHelper: discoveryHelper, + backupper: backupper, + lister: backupInformer.Lister(), + client: client, + clock: &clocks.RealClock{}, + backupLogLevel: backupLogLevel, + newPluginManager: newPluginManager, + backupTracker: backupTracker, + kbClient: kbClient, + defaultBackupLocation: defaultBackupLocation, + defaultVolumesToFsBackup: defaultVolumesToFsBackup, + defaultBackupTTL: defaultBackupTTL, + defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, + defaultItemOperationTimeout: defaultItemOperationTimeout, + snapshotLocationLister: volumeSnapshotLocationLister, + defaultSnapshotLocations: defaultSnapshotLocations, + metrics: metrics, + backupStoreGetter: backupStoreGetter, + formatFlag: formatFlag, + volumeSnapshotLister: volumeSnapshotLister, + volumeSnapshotClient: volumeSnapshotClient, + credentialFileStore: credentialStore, } c.syncHandler = c.processBackup @@ -366,6 +369,11 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logg request.Spec.CSISnapshotTimeout.Duration = c.defaultCSISnapshotTimeout } + if request.Spec.ItemOperationTimeout.Duration == 0 { + // set default item operation timeout + request.Spec.ItemOperationTimeout.Duration = c.defaultItemOperationTimeout + } + // calculate expiration request.Status.Expiration = &metav1.Time{Time: c.clock.Now().Add(request.Spec.TTL.Duration)} @@ -705,10 +713,6 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { } } - // Mark completion timestamp before serializing and uploading. - // Otherwise, the JSON file in object storage has a CompletionTimestamp of 'null'. - backup.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} - backup.Status.VolumeSnapshotsAttempted = len(backup.VolumeSnapshots) for _, snap := range backup.VolumeSnapshots { if snap.Status.Phase == volume.SnapshotPhaseCompleted { @@ -723,11 +727,24 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { } } + // Iterate over backup item operations and update progress. + // Any errors on operations at this point should be added to backup errors. + // If any operations are still not complete, then back will not be set to + // Completed yet. + inProgressOperations, _, opsCompleted, opsFailed, errs := getBackupItemOperationProgress(backup.Backup, pluginManager, *backup.GetItemOperationsList()) + if len(errs) > 0 { + for err := range errs { + backupLog.Error(err) + } + } + + backup.Status.AsyncBackupItemOperationsAttempted = len(*backup.GetItemOperationsList()) + backup.Status.AsyncBackupItemOperationsCompleted = opsCompleted + backup.Status.AsyncBackupItemOperationsFailed = opsFailed + backup.Status.Warnings = logCounter.GetCount(logrus.WarnLevel) backup.Status.Errors = logCounter.GetCount(logrus.ErrorLevel) - recordBackupMetrics(backupLog, backup.Backup, backupFile, c.metrics) - backupWarnings := logCounter.GetEntries(logrus.WarnLevel) backupErrors := logCounter.GetEntries(logrus.ErrorLevel) results := map[string]results.Result{ @@ -747,10 +764,26 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { case len(fatalErrs) > 0: backup.Status.Phase = velerov1api.BackupPhaseFailed case logCounter.GetCount(logrus.ErrorLevel) > 0: - backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + if inProgressOperations { + backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed + } else { + backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed + } default: - backup.Status.Phase = velerov1api.BackupPhaseCompleted + if inProgressOperations { + backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperations + } else { + backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperations + } + } + // Mark completion timestamp before serializing and uploading. + // Otherwise, the JSON file in object storage has a CompletionTimestamp of 'null'. + if backup.Status.Phase == velerov1api.BackupPhaseFailed || + backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || + backup.Status.Phase == velerov1api.BackupPhaseCompleted { + backup.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} } + recordBackupMetrics(backupLog, backup.Backup, backupFile, c.metrics, false) // re-instantiate the backup store because credentials could have changed since the original // instantiation, if this was a long-running backup @@ -771,37 +804,43 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { return kerrors.NewAggregate(fatalErrs) } -func recordBackupMetrics(log logrus.FieldLogger, backup *velerov1api.Backup, backupFile *os.File, serverMetrics *metrics.ServerMetrics) { +func recordBackupMetrics(log logrus.FieldLogger, backup *velerov1api.Backup, backupFile *os.File, serverMetrics *metrics.ServerMetrics, finalize bool) { backupScheduleName := backup.GetLabels()[velerov1api.ScheduleNameLabel] - var backupSizeBytes int64 - if backupFileStat, err := backupFile.Stat(); err != nil { - log.WithError(errors.WithStack(err)).Error("Error getting backup file info") - } else { - backupSizeBytes = backupFileStat.Size() + if backupFile != nil { + var backupSizeBytes int64 + if backupFileStat, err := backupFile.Stat(); err != nil { + log.WithError(errors.WithStack(err)).Error("Error getting backup file info") + } else { + backupSizeBytes = backupFileStat.Size() + } + serverMetrics.SetBackupTarballSizeBytesGauge(backupScheduleName, backupSizeBytes) } - serverMetrics.SetBackupTarballSizeBytesGauge(backupScheduleName, backupSizeBytes) - - backupDuration := backup.Status.CompletionTimestamp.Time.Sub(backup.Status.StartTimestamp.Time) - backupDurationSeconds := float64(backupDuration / time.Second) - serverMetrics.RegisterBackupDuration(backupScheduleName, backupDurationSeconds) - serverMetrics.RegisterVolumeSnapshotAttempts(backupScheduleName, backup.Status.VolumeSnapshotsAttempted) - serverMetrics.RegisterVolumeSnapshotSuccesses(backupScheduleName, backup.Status.VolumeSnapshotsCompleted) - serverMetrics.RegisterVolumeSnapshotFailures(backupScheduleName, backup.Status.VolumeSnapshotsAttempted-backup.Status.VolumeSnapshotsCompleted) - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - serverMetrics.RegisterCSISnapshotAttempts(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsAttempted) - serverMetrics.RegisterCSISnapshotSuccesses(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsCompleted) - serverMetrics.RegisterCSISnapshotFailures(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsAttempted-backup.Status.CSIVolumeSnapshotsCompleted) + if backup.Status.CompletionTimestamp != nil { + backupDuration := backup.Status.CompletionTimestamp.Time.Sub(backup.Status.StartTimestamp.Time) + backupDurationSeconds := float64(backupDuration / time.Second) + serverMetrics.RegisterBackupDuration(backupScheduleName, backupDurationSeconds) } + if !finalize { + serverMetrics.RegisterVolumeSnapshotAttempts(backupScheduleName, backup.Status.VolumeSnapshotsAttempted) + serverMetrics.RegisterVolumeSnapshotSuccesses(backupScheduleName, backup.Status.VolumeSnapshotsCompleted) + serverMetrics.RegisterVolumeSnapshotFailures(backupScheduleName, backup.Status.VolumeSnapshotsAttempted-backup.Status.VolumeSnapshotsCompleted) - if backup.Status.Progress != nil { - serverMetrics.RegisterBackupItemsTotalGauge(backupScheduleName, backup.Status.Progress.TotalItems) - } - serverMetrics.RegisterBackupItemsErrorsGauge(backupScheduleName, backup.Status.Errors) + if features.IsEnabled(velerov1api.CSIFeatureFlag) { + serverMetrics.RegisterCSISnapshotAttempts(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsAttempted) + serverMetrics.RegisterCSISnapshotSuccesses(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsCompleted) + serverMetrics.RegisterCSISnapshotFailures(backupScheduleName, backup.Name, backup.Status.CSIVolumeSnapshotsAttempted-backup.Status.CSIVolumeSnapshotsCompleted) + } + + if backup.Status.Progress != nil { + serverMetrics.RegisterBackupItemsTotalGauge(backupScheduleName, backup.Status.Progress.TotalItems) + } + serverMetrics.RegisterBackupItemsErrorsGauge(backupScheduleName, backup.Status.Errors) - if backup.Status.Warnings > 0 { - serverMetrics.RegisterBackupWarning(backupScheduleName) + if backup.Status.Warnings > 0 { + serverMetrics.RegisterBackupWarning(backupScheduleName) + } } } @@ -826,6 +865,12 @@ func persistBackup(backup *pkgbackup.Request, persistErrs = append(persistErrs, errs...) } + var backupItemOperations *bytes.Buffer + backupItemOperations, errs = encodeToJSONGzip(backup.GetItemOperationsList(), "backup item operations list") + if errs != nil { + persistErrs = append(persistErrs, errs...) + } + podVolumeBackups, errs := encodeToJSONGzip(backup.PodVolumeBackups, "pod volume backups list") if errs != nil { persistErrs = append(persistErrs, errs...) @@ -860,6 +905,7 @@ func persistBackup(backup *pkgbackup.Request, backupJSON = nil backupContents = nil nativeVolumeSnapshots = nil + backupItemOperations = nil backupResourceList = nil csiSnapshotJSON = nil csiSnapshotContentsJSON = nil @@ -875,6 +921,7 @@ func persistBackup(backup *pkgbackup.Request, BackupResults: backupResult, PodVolumeBackups: podVolumeBackups, VolumeSnapshots: nativeVolumeSnapshots, + BackupItemOperations: backupItemOperations, BackupResourceList: backupResourceList, CSIVolumeSnapshots: csiSnapshotJSON, CSIVolumeSnapshotContents: csiSnapshotContentsJSON, diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 0c7949781c..9c90da37e8 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -47,6 +47,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" @@ -75,6 +76,13 @@ func (b *fakeBackupper) BackupWithResolvers(logger logrus.FieldLogger, backup *p return args.Error(0) } +func (b *fakeBackupper) FinalizeBackup(logger logrus.FieldLogger, backup *pkgbackup.Request, inBackupFile io.Reader, outBackupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolverV2, + asyncBIAOperations []*itemoperation.BackupOperation) error { + args := b.Called(logger, backup, inBackupFile, outBackupFile, backupItemActionResolver, asyncBIAOperations) + return args.Error(0) +} + func defaultBackup() *builder.BackupBuilder { return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1") } @@ -597,7 +605,7 @@ func TestProcessBackupCompletions(t *testing.T) { backupExists bool existenceCheckError error }{ - // Completed + // FinalizingAfterPluginOperations { name: "backup with no backup location gets the default", backup: defaultBackup().Result(), @@ -625,12 +633,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, - Expiration: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, }, }, }, @@ -661,12 +668,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.False(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, - Expiration: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, }, }, }, @@ -700,12 +706,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, - Expiration: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, }, }, }, @@ -737,12 +742,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.False(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - Expiration: &metav1.Time{now.Add(10 * time.Minute)}, - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + Expiration: &metav1.Time{now.Add(10 * time.Minute)}, + StartTimestamp: ×tamp, }, }, }, @@ -774,12 +778,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, - Expiration: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, }, }, }, @@ -812,12 +815,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.False(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, - Expiration: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, }, }, }, @@ -850,12 +852,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, - Expiration: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, }, }, }, @@ -888,12 +889,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.True(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, - Expiration: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, }, }, }, @@ -926,12 +926,11 @@ func TestProcessBackupCompletions(t *testing.T) { DefaultVolumesToFsBackup: boolptr.False(), }, Status: velerov1api.BackupStatus{ - Phase: velerov1api.BackupPhaseCompleted, - Version: 1, - FormatVersion: "1.1.0", - StartTimestamp: ×tamp, - CompletionTimestamp: ×tamp, - Expiration: ×tamp, + Phase: velerov1api.BackupPhaseFinalizingAfterPluginOperations, + Version: 1, + FormatVersion: "1.1.0", + StartTimestamp: ×tamp, + Expiration: ×tamp, }, }, }, @@ -1079,13 +1078,16 @@ func TestProcessBackupCompletions(t *testing.T) { // Ensure we have a CompletionTimestamp when uploading and that the backup name matches the backup in the object store. // Failures will display the bytes in buf. - hasNameAndCompletionTimestamp := func(info persistence.BackupInfo) bool { + hasNameAndCompletionTimestampIfCompleted := func(info persistence.BackupInfo) bool { buf := new(bytes.Buffer) buf.ReadFrom(info.Metadata) return info.Name == test.backup.Name && - strings.Contains(buf.String(), `"completionTimestamp": "2006-01-02T22:04:05Z"`) + (!(strings.Contains(buf.String(), `"phase": "Completed"`) || + strings.Contains(buf.String(), `"phase": "Failed"`) || + strings.Contains(buf.String(), `"phase": "PartiallyFailed"`)) || + strings.Contains(buf.String(), `"completionTimestamp": "2006-01-02T22:04:05Z"`)) } - backupStore.On("PutBackup", mock.MatchedBy(hasNameAndCompletionTimestamp)).Return(nil) + backupStore.On("PutBackup", mock.MatchedBy(hasNameAndCompletionTimestampIfCompleted)).Return(nil) // add the test's backup to the informer/lister store require.NotNil(t, test.backup) diff --git a/pkg/controller/backup_finalizer_controller.go b/pkg/controller/backup_finalizer_controller.go new file mode 100644 index 0000000000..c8db402bab --- /dev/null +++ b/pkg/controller/backup_finalizer_controller.go @@ -0,0 +1,204 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "io/ioutil" + "os" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clocks "k8s.io/utils/clock" + ctrl "sigs.k8s.io/controller-runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" + "github.com/vmware-tanzu/velero/pkg/metrics" + "github.com/vmware-tanzu/velero/pkg/persistence" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/vmware-tanzu/velero/pkg/util/encode" +) + +// backupFinalizerReconciler reconciles a Backup object +type backupFinalizerReconciler struct { + client kbclient.Client + clock clocks.WithTickerAndDelayedExecution + backupper pkgbackup.Backupper + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + metrics *metrics.ServerMetrics + backupStoreGetter persistence.ObjectBackupStoreGetter + log logrus.FieldLogger +} + +// NewBackupFinalizerReconciler initializes and returns backupFinalizerReconciler struct. +func NewBackupFinalizerReconciler( + client kbclient.Client, + clock clocks.WithTickerAndDelayedExecution, + backupper pkgbackup.Backupper, + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, + backupStoreGetter persistence.ObjectBackupStoreGetter, + log logrus.FieldLogger, + metrics *metrics.ServerMetrics, +) *backupFinalizerReconciler { + return &backupFinalizerReconciler{ + client: client, + clock: clock, + backupper: backupper, + newPluginManager: newPluginManager, + backupStoreGetter: backupStoreGetter, + log: log, + metrics: metrics, + } +} + +// +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get;update;patch +func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.log.WithFields(logrus.Fields{ + "controller": "backup-finalizer", + "backup": req.NamespacedName, + }) + + // Fetch the Backup instance. + log.Debug("Getting Backup") + backup := &velerov1api.Backup{} + if err := r.client.Get(ctx, req.NamespacedName, backup); err != nil { + if apierrors.IsNotFound(err) { + log.Debug("Unable to find Backup") + return ctrl.Result{}, nil + } + + log.WithError(err).Error("Error getting Backup") + return ctrl.Result{}, errors.WithStack(err) + } + + switch backup.Status.Phase { + case velerov1api.BackupPhaseFinalizingAfterPluginOperations, velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed: + // only process backups finalizing after plugin operations are complete + default: + log.Debug("Backup is not awaiting finalizing, skipping") + return ctrl.Result{}, nil + } + + original := backup.DeepCopy() + defer func() { + // Always attempt to Patch the backup object and status after each reconciliation. + if err := r.client.Patch(ctx, backup, kbclient.MergeFrom(original)); err != nil { + log.WithError(err).Error("Error updating backup") + return + } + }() + + location := &velerov1api.BackupStorageLocation{} + if err := r.client.Get(ctx, kbclient.ObjectKey{ + Namespace: backup.Namespace, + Name: backup.Spec.StorageLocation, + }, location); err != nil { + return ctrl.Result{}, errors.WithStack(err) + } + pluginManager := r.newPluginManager(log) + defer pluginManager.CleanupClients() + + backupStore, err := r.backupStoreGetter.Get(location, pluginManager, log) + if err != nil { + log.WithError(err).Error("Error getting a backup store") + return ctrl.Result{}, errors.WithStack(err) + } + + // Download item operations list and backup contents + operations, err := backupStore.GetBackupItemOperations(backup.Name) + if err != nil { + log.WithError(err).Error("Error getting backup item operations") + return ctrl.Result{}, errors.WithStack(err) + } + + backupRequest := &pkgbackup.Request{ + Backup: backup, + StorageLocation: location, + } + var outBackupFile *os.File + if len(operations) > 0 { + // Call itemBackupper.BackupItem for the list of items updated by async operations + log.Info("Setting up finalized backup temp file") + inBackupFile, err := downloadToTempFile(backup.Name, backupStore, log) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "error downloading backup") + } + defer closeAndRemoveFile(inBackupFile, log) + outBackupFile, err = ioutil.TempFile("", "") + if err != nil { + log.WithError(err).Error("error creating temp file for backup") + return ctrl.Result{}, errors.WithStack(err) + } + defer closeAndRemoveFile(outBackupFile, log) + + log.Info("Getting backup item actions") + actions, err := pluginManager.GetBackupItemActionsV2() + if err != nil { + log.WithError(err).Error("error getting Backup Item Actions") + return ctrl.Result{}, errors.WithStack(err) + } + backupItemActionsResolver := framework.NewBackupItemActionResolverV2(actions) + err = r.backupper.FinalizeBackup(log, backupRequest, inBackupFile, outBackupFile, backupItemActionsResolver, operations) + if err != nil { + log.WithError(err).Error("error finalizing Backup") + return ctrl.Result{}, errors.WithStack(err) + } + } + backupScheduleName := backupRequest.GetLabels()[velerov1api.ScheduleNameLabel] + switch backup.Status.Phase { + case velerov1api.BackupPhaseFinalizingAfterPluginOperations: + backup.Status.Phase = velerov1api.BackupPhaseCompleted + r.metrics.RegisterBackupSuccess(backupScheduleName) + r.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusSucc) + case velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed: + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + r.metrics.RegisterBackupPartialFailure(backupScheduleName) + r.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure) + } + backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} + recordBackupMetrics(log, backup, outBackupFile, r.metrics, true) + + // update backup metadata in object store + backupJSON := new(bytes.Buffer) + if err := encode.EncodeTo(backup, "json", backupJSON); err != nil { + return ctrl.Result{}, errors.Wrap(err, "error encoding backup json") + } + err = backupStore.PutBackupMetadata(backup.Name, backupJSON) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "error uploading backup json") + } + if len(operations) > 0 { + err = backupStore.PutBackupContents(backup.Name, outBackupFile) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "error uploading backup final contents") + } + } + return ctrl.Result{}, nil +} + +func (r *backupFinalizerReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&velerov1api.Backup{}). + Complete(r) +} diff --git a/pkg/controller/backup_finalizer_controller_test.go b/pkg/controller/backup_finalizer_controller_test.go new file mode 100644 index 0000000000..828412a7f9 --- /dev/null +++ b/pkg/controller/backup_finalizer_controller_test.go @@ -0,0 +1,184 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "io/ioutil" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + testclocks "k8s.io/utils/clock/testing" + ctrl "sigs.k8s.io/controller-runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/metrics" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) { + backupper := new(fakeBackupper) + return NewBackupFinalizerReconciler( + fakeClient, + fakeClock, + backupper, + func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewFakeSingleObjectBackupStoreGetter(backupStore), + logrus.StandardLogger(), + metrics.NewServerMetrics(), + ), backupper +} +func TestBackupFinalizerReconcile(t *testing.T) { + fakeClock := testclocks.NewFakeClock(time.Now()) + metav1Now := metav1.NewTime(fakeClock.Now()) + + defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Result() + + tests := []struct { + name string + backup *velerov1api.Backup + backupOperations []*itemoperation.BackupOperation + backupLocation *velerov1api.BackupStorageLocation + expectError bool + expectPhase velerov1api.BackupPhase + }{ + { + name: "FinalizingAfterPluginOperations backup is completed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + StorageLocation("default"). + ObjectMeta(builder.WithUID("foo")). + StartTimestamp(fakeClock.Now()). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations).Result(), + backupLocation: defaultBackupLocation, + expectPhase: velerov1api.BackupPhaseCompleted, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-1", + BackupUID: "foo", + BackupItemAction: "foo", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + ItemsToUpdate: []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.Secrets, + Namespace: "ns-1", + Name: "secret-1", + }, + }, + OperationID: "operation-1", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseCompleted, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "FinalizingAfterPluginOperationsPartiallyFailed backup is partially failed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). + StorageLocation("default"). + ObjectMeta(builder.WithUID("foo")). + StartTimestamp(fakeClock.Now()). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed).Result(), + backupLocation: defaultBackupLocation, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-2", + BackupUID: "foo", + BackupItemAction: "foo", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-2", + Name: "pod-2", + }, + ItemsToUpdate: []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.Secrets, + Namespace: "ns-2", + Name: "secret-2", + }, + }, + OperationID: "operation-2", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseCompleted, + Created: &metav1Now, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.backup == nil { + return + } + + initObjs := []runtime.Object{} + initObjs = append(initObjs, test.backup) + + if test.backupLocation != nil { + initObjs = append(initObjs, test.backupLocation) + } + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) + reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeClock) + pluginManager.On("CleanupClients").Return(nil) + backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil) + backupStore.On("GetBackupContents", mock.Anything).Return(ioutil.NopCloser(bytes.NewReader([]byte("hello world"))), nil) + backupStore.On("PutBackupContents", mock.Anything, mock.Anything).Return(nil) + backupStore.On("PutBackupMetadata", mock.Anything, mock.Anything).Return(nil) + pluginManager.On("GetBackupItemActionsV2").Return(nil, nil) + backupper.On("FinalizeBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, framework.BackupItemActionResolverV2{}, mock.Anything).Return(nil) + _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) + gotErr := err != nil + assert.Equal(t, test.expectError, gotErr) + + backupAfter := velerov1api.Backup{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{ + Namespace: test.backup.Namespace, + Name: test.backup.Name, + }, &backupAfter) + + require.NoError(t, err) + assert.Equal(t, test.expectPhase, backupAfter.Status.Phase) + }) + } +} diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index d29a586416..e8114abcf8 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -148,6 +148,26 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) continue } + if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations || + backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed { + + if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { + log.Debugf("Skipping non-expired WaitingForPluginOperations backup %v", backup.Name) + continue + } + log.Debug("WaitingForPluginOperations Backup is past expiration, syncing for garbage collection") + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } + if backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed { + + if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { + log.Debugf("Skipping non-expired FinalizingAfterPluginOperations backup %v", backup.Name) + continue + } + log.Debug("FinalizingAfterPluginOperations Backup is past expiration, syncing for garbage collection") + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } backup.Namespace = b.namespace backup.ResourceVersion = "" diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index 4f1e280c55..44d40b729c 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -32,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" core "k8s.io/client-go/testing" + testclocks "k8s.io/utils/clock/testing" ctrl "sigs.k8s.io/controller-runtime" ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" @@ -155,9 +157,11 @@ func numBackups(c ctrlClient.WithWatch, ns string) (int, error) { var _ = Describe("Backup Sync Reconciler", func() { It("Test Backup Sync Reconciler basic function", func() { + fakeClock := testclocks.NewFakeClock(time.Now()) type cloudBackupData struct { - backup *velerov1api.Backup - podVolumeBackups []*velerov1api.PodVolumeBackup + backup *velerov1api.Backup + podVolumeBackups []*velerov1api.PodVolumeBackup + backupShouldSkipSync bool // backups waiting for plugin operations should not sync } tests := []struct { @@ -187,6 +191,98 @@ var _ = Describe("Backup Sync Reconciler", func() { }, }, }, + { + name: "backups waiting for plugin operations aren't synced", + namespace: "ns-1", + location: defaultLocation("ns-1"), + cloudBackups: []*cloudBackupData{ + { + backup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-2"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-3"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), + }, + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-4"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-5"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-6"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations).Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), + }, + backupShouldSkipSync: true, + }, + }, + }, + { + name: "expired backups waiting for plugin operations are synced", + namespace: "ns-1", + location: defaultLocation("ns-1"), + cloudBackups: []*cloudBackupData{ + { + backup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-2"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-3"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), + }, + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-4"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-5"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-6"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), + }, + backupShouldSkipSync: true, + }, + }, + }, { name: "all synced backups get created in Velero server's namespace", namespace: "velero", @@ -364,36 +460,42 @@ var _ = Describe("Backup Sync Reconciler", func() { Namespace: cloudBackupData.backup.Namespace, Name: cloudBackupData.backup.Name}, obj) - Expect(err).To(BeNil()) - - // did this cloud backup already exist in the cluster? - var existing *velerov1api.Backup - for _, obj := range test.existingBackups { - if obj.Name == cloudBackupData.backup.Name { - existing = obj - break + if cloudBackupData.backupShouldSkipSync && + (cloudBackupData.backup.Status.Expiration == nil || + cloudBackupData.backup.Status.Expiration.After(fakeClock.Now())) { + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } else { + Expect(err).To(BeNil()) + + // did this cloud backup already exist in the cluster? + var existing *velerov1api.Backup + for _, obj := range test.existingBackups { + if obj.Name == cloudBackupData.backup.Name { + existing = obj + break + } } - } - if existing != nil { - // if this cloud backup already exists in the cluster, make sure that what we get from the - // client is the existing backup, not the cloud one. + if existing != nil { + // if this cloud backup already exists in the cluster, make sure that what we get from the + // client is the existing backup, not the cloud one. - // verify that the in-cluster backup has its storage location populated, if it's not already. - expected := existing.DeepCopy() - expected.Spec.StorageLocation = test.location.Name + // verify that the in-cluster backup has its storage location populated, if it's not already. + expected := existing.DeepCopy() + expected.Spec.StorageLocation = test.location.Name - Expect(expected).To(BeEquivalentTo(obj)) - } else { - // verify that the storage location field and label are set properly - Expect(test.location.Name).To(BeEquivalentTo(obj.Spec.StorageLocation)) + Expect(expected).To(BeEquivalentTo(obj)) + } else { + // verify that the storage location field and label are set properly + Expect(test.location.Name).To(BeEquivalentTo(obj.Spec.StorageLocation)) - locationName := test.location.Name - if test.longLocationNameEnabled { - locationName = label.GetValidName(locationName) + locationName := test.location.Name + if test.longLocationNameEnabled { + locationName = label.GetValidName(locationName) + } + Expect(locationName).To(BeEquivalentTo(obj.Labels[velerov1api.StorageLocationLabel])) + Expect(len(obj.Labels[velerov1api.StorageLocationLabel]) <= validation.DNS1035LabelMaxLength).To(BeTrue()) } - Expect(locationName).To(BeEquivalentTo(obj.Labels[velerov1api.StorageLocationLabel])) - Expect(len(obj.Labels[velerov1api.StorageLocationLabel]) <= validation.DNS1035LabelMaxLength).To(BeTrue()) } // process the cloud pod volume backups for this backup, if any @@ -406,22 +508,28 @@ var _ = Describe("Backup Sync Reconciler", func() { Name: podVolumeBackup.Name, }, objPodVolumeBackup) - Expect(err).ShouldNot(HaveOccurred()) - - // did this cloud pod volume backup already exist in the cluster? - var existingPodVolumeBackup *velerov1api.PodVolumeBackup - for _, objPodVolumeBackup := range test.existingPodVolumeBackups { - if objPodVolumeBackup.Name == podVolumeBackup.Name { - existingPodVolumeBackup = objPodVolumeBackup - break + if cloudBackupData.backupShouldSkipSync && + (cloudBackupData.backup.Status.Expiration == nil || + cloudBackupData.backup.Status.Expiration.After(fakeClock.Now())) { + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } else { + Expect(err).ShouldNot(HaveOccurred()) + + // did this cloud pod volume backup already exist in the cluster? + var existingPodVolumeBackup *velerov1api.PodVolumeBackup + for _, objPodVolumeBackup := range test.existingPodVolumeBackups { + if objPodVolumeBackup.Name == podVolumeBackup.Name { + existingPodVolumeBackup = objPodVolumeBackup + break + } } - } - if existingPodVolumeBackup != nil { - // if this cloud pod volume backup already exists in the cluster, make sure that what we get from the - // client is the existing backup, not the cloud one. - expected := existingPodVolumeBackup.DeepCopy() - Expect(expected).To(BeEquivalentTo(objPodVolumeBackup)) + if existingPodVolumeBackup != nil { + // if this cloud pod volume backup already exists in the cluster, make sure that what we get from the + // client is the existing backup, not the cloud one. + expected := existingPodVolumeBackup.DeepCopy() + Expect(expected).To(BeEquivalentTo(objPodVolumeBackup)) + } } } } diff --git a/pkg/controller/constants.go b/pkg/controller/constants.go index 55d1ac7653..2d051e0b53 100644 --- a/pkg/controller/constants.go +++ b/pkg/controller/constants.go @@ -17,8 +17,10 @@ limitations under the License. package controller const ( + AsyncBackupOperations = "async-backup-operations" Backup = "backup" BackupDeletion = "backup-deletion" + BackupFinalizer = "backup-finalizer" BackupStorageLocation = "backup-storage-location" BackupSync = "backup-sync" DownloadRequest = "download-request" @@ -33,8 +35,10 @@ const ( // DisableableControllers is a list of controllers that can be disabled var DisableableControllers = []string{ + AsyncBackupOperations, Backup, BackupDeletion, + BackupFinalizer, BackupSync, DownloadRequest, GarbageCollection, diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index b35dd9aadb..45491aba68 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -41,6 +41,9 @@ type downloadRequestReconciler struct { newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter + // used to force update of async backup item operations before processing download request + backupItemOperationsMap *BackupItemOperationsMap + log logrus.FieldLogger } @@ -51,13 +54,15 @@ func NewDownloadRequestReconciler( newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, log logrus.FieldLogger, + backupItemOperationsMap *BackupItemOperationsMap, ) *downloadRequestReconciler { return &downloadRequestReconciler{ - client: client, - clock: clock, - newPluginManager: newPluginManager, - backupStoreGetter: backupStoreGetter, - log: log, + client: client, + clock: clock, + newPluginManager: newPluginManager, + backupStoreGetter: backupStoreGetter, + backupItemOperationsMap: backupItemOperationsMap, + log: log, } } @@ -158,6 +163,13 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, errors.WithStack(err) } + // If this is a request for backup item operations, force update of in-memory operations that + // are not yet uploaded + if downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindBackupItemOperations && + r.backupItemOperationsMap != nil { + // ignore errors here. If we can't upload anything here, process the download as usual + _ = r.backupItemOperationsMap.UpdateForBackup(backupStore, backupName) + } if downloadRequest.Status.DownloadURL, err = backupStore.GetDownloadURL(downloadRequest.Spec.Target); err != nil { return ctrl.Result{Requeue: true}, errors.WithStack(err) } diff --git a/pkg/controller/download_request_controller_test.go b/pkg/controller/download_request_controller_test.go index 1cbf1be03e..01f67414fe 100644 --- a/pkg/controller/download_request_controller_test.go +++ b/pkg/controller/download_request_controller_test.go @@ -112,6 +112,7 @@ var _ = Describe("Download Request Reconciler", func() { func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeObjectBackupStoreGetter(backupStores), velerotest.NewLogger(), + nil, ) if test.backupLocation != nil && test.expectGetsURL { diff --git a/pkg/itemoperation/backup_operation.go b/pkg/itemoperation/backup_operation.go index 94979ceeb9..5c226f13e4 100644 --- a/pkg/itemoperation/backup_operation.go +++ b/pkg/itemoperation/backup_operation.go @@ -16,6 +16,10 @@ limitations under the License. package itemoperation +import ( + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + // BackupOperation stores information about an async item operation // started by a BackupItemAction plugin (v2 or later) type BackupOperation struct { @@ -24,6 +28,21 @@ type BackupOperation struct { Status OperationStatus `json:"status"` } +func (in *BackupOperation) DeepCopy() *BackupOperation { + if in == nil { + return nil + } + out := new(BackupOperation) + in.DeepCopyInto(out) + return out +} + +func (in *BackupOperation) DeepCopyInto(out *BackupOperation) { + *out = *in + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + type BackupOperationSpec struct { // BackupName is the name of the Velero backup this item operation // is associated with. @@ -37,8 +56,32 @@ type BackupOperationSpec struct { BackupItemAction string `json:"backupItemAction"` // Kubernetes resource identifier for the item - ResourceIdentifier string "json:resourceIdentifier" + ResourceIdentifier velero.ResourceIdentifier "json:resourceIdentifier" // OperationID returned by the BIA plugin OperationID string "json:operationID" + + // Items needing update after all async operations have completed + ItemsToUpdate []velero.ResourceIdentifier "json:itemsToUpdate" +} + +func (in *BackupOperationSpec) DeepCopy() *BackupOperationSpec { + if in == nil { + return nil + } + out := new(BackupOperationSpec) + in.DeepCopyInto(out) + return out +} + +func (in *BackupOperationSpec) DeepCopyInto(out *BackupOperationSpec) { + *out = *in + in.ResourceIdentifier.DeepCopyInto(&out.ResourceIdentifier) + if in.ItemsToUpdate != nil { + in, out := &in.ItemsToUpdate, &out.ItemsToUpdate + *out = make([]velero.ResourceIdentifier, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } diff --git a/pkg/itemoperation/restore_operation.go b/pkg/itemoperation/restore_operation.go index e3fe4d1f00..50894983d1 100644 --- a/pkg/itemoperation/restore_operation.go +++ b/pkg/itemoperation/restore_operation.go @@ -16,6 +16,10 @@ limitations under the License. package itemoperation +import ( + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + // RestoreOperation stores information about an async item operation // started by a RestoreItemAction plugin (v2 or later) type RestoreOperation struct { @@ -37,7 +41,7 @@ type RestoreOperationSpec struct { RestoreItemAction string `json:"restoreItemAction"` // Kubernetes resource identifier for the item - ResourceIdentifier string "json:resourceIdentifier" + ResourceIdentifier velero.ResourceIdentifier "json:resourceIdentifier" // OperationID returned by the RIA plugin OperationID string "json:operationID" diff --git a/pkg/itemoperation/shared.go b/pkg/itemoperation/shared.go index b9039b3ba2..b9c81ce0ba 100644 --- a/pkg/itemoperation/shared.go +++ b/pkg/itemoperation/shared.go @@ -40,12 +40,19 @@ type OperationStatus struct { // Units that NCompleted,NTotal are measured in // i.e. "bytes" - OperationUnits int64 `json:"nTotal,omitempty"` + OperationUnits string `json:"operationUnits,omitempty"` + + // Description of progress made + // i.e. "processing", "Current phase: Running", etc. + Description string `json:"description,omitempty"` + + // Created records the time the item operation was created + Created *metav1.Time `json:"created,omitempty"` // Started records the time the item operation was started, if known // +optional // +nullable - Started *metav1.Time `json:"start,omitempty"` + Started *metav1.Time `json:"started,omitempty"` // Updated records the time the item operation was updated, if known. // +optional @@ -53,10 +60,35 @@ type OperationStatus struct { Updated *metav1.Time `json:"updated,omitempty"` } +func (in *OperationStatus) DeepCopy() *OperationStatus { + if in == nil { + return nil + } + out := new(OperationStatus) + in.DeepCopyInto(out) + return out +} + +func (in *OperationStatus) DeepCopyInto(out *OperationStatus) { + *out = *in + if in.Created != nil { + in, out := &in.Created, &out.Created + *out = (*in).DeepCopy() + } + if in.Started != nil { + in, out := &in.Started, &out.Started + *out = (*in).DeepCopy() + } + if in.Updated != nil { + in, out := &in.Updated, &out.Updated + *out = (*in).DeepCopy() + } +} + const ( // OperationPhaseNew means the item operation has been created and started // by the plugin - OperationPhaseInProgress OperationPhase = "New" + OperationPhaseInProgress OperationPhase = "InProgress" // OperationPhaseCompleted means the item operation was successfully completed // and can be used for restore. diff --git a/pkg/persistence/mocks/backup_store.go b/pkg/persistence/mocks/backup_store.go index 033ba2ecea..7d71782262 100644 --- a/pkg/persistence/mocks/backup_store.go +++ b/pkg/persistence/mocks/backup_store.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +// Code generated by mockery v2.16.0. DO NOT EDIT. package mocks @@ -20,13 +21,15 @@ import ( io "io" mock "github.com/stretchr/testify/mock" + itemoperation "github.com/vmware-tanzu/velero/pkg/itemoperation" - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + persistence "github.com/vmware-tanzu/velero/pkg/persistence" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/itemoperation" - persistence "github.com/vmware-tanzu/velero/pkg/persistence" + volume "github.com/vmware-tanzu/velero/pkg/volume" + + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" ) // BackupStore is an autogenerated mock type for the BackupStore type @@ -106,6 +109,29 @@ func (_m *BackupStore) GetBackupContents(name string) (io.ReadCloser, error) { return r0, r1 } +// GetBackupItemOperations provides a mock function with given fields: name +func (_m *BackupStore) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) { + ret := _m.Called(name) + + var r0 []*itemoperation.BackupOperation + if rf, ok := ret.Get(0).(func(string) []*itemoperation.BackupOperation); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*itemoperation.BackupOperation) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetBackupMetadata provides a mock function with given fields: name func (_m *BackupStore) GetBackupMetadata(name string) (*v1.Backup, error) { ret := _m.Called(name) @@ -152,6 +178,75 @@ func (_m *BackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot return r0, r1 } +// GetCSIVolumeSnapshotClasses provides a mock function with given fields: name +func (_m *BackupStore) GetCSIVolumeSnapshotClasses(name string) ([]*volumesnapshotv1.VolumeSnapshotClass, error) { + ret := _m.Called(name) + + var r0 []*volumesnapshotv1.VolumeSnapshotClass + if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshotClass); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshotClass) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetCSIVolumeSnapshotContents provides a mock function with given fields: name +func (_m *BackupStore) GetCSIVolumeSnapshotContents(name string) ([]*volumesnapshotv1.VolumeSnapshotContent, error) { + ret := _m.Called(name) + + var r0 []*volumesnapshotv1.VolumeSnapshotContent + if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshotContent); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshotContent) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetCSIVolumeSnapshots provides a mock function with given fields: name +func (_m *BackupStore) GetCSIVolumeSnapshots(name string) ([]*volumesnapshotv1.VolumeSnapshot, error) { + ret := _m.Called(name) + + var r0 []*volumesnapshotv1.VolumeSnapshot + if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshot); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshot) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetDownloadURL provides a mock function with given fields: target func (_m *BackupStore) GetDownloadURL(target v1.DownloadTarget) (string, error) { ret := _m.Called(target) @@ -196,6 +291,29 @@ func (_m *BackupStore) GetPodVolumeBackups(name string) ([]*v1.PodVolumeBackup, return r0, r1 } +// GetRestoreItemOperations provides a mock function with given fields: name +func (_m *BackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.RestoreOperation, error) { + ret := _m.Called(name) + + var r0 []*itemoperation.RestoreOperation + if rf, ok := ret.Get(0).(func(string) []*itemoperation.RestoreOperation); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*itemoperation.RestoreOperation) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // IsValid provides a mock function with given fields: func (_m *BackupStore) IsValid() error { ret := _m.Called() @@ -247,13 +365,13 @@ func (_m *BackupStore) PutBackup(info persistence.BackupInfo) error { return r0 } -// PutRestoreLog provides a mock function with given fields: backup, restore, log -func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error { - ret := _m.Called(backup, restore, log) +// PutBackupContents provides a mock function with given fields: backup, backupContents +func (_m *BackupStore) PutBackupContents(backup string, backupContents io.Reader) error { + ret := _m.Called(backup, backupContents) var r0 error - if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { - r0 = rf(backup, restore, log) + if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { + r0 = rf(backup, backupContents) } else { r0 = ret.Error(0) } @@ -261,13 +379,27 @@ func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reade return r0 } -// PutRestoreResults provides a mock function with given fields: backup, restore, results -func (_m *BackupStore) PutRestoreResults(backup string, restore string, results io.Reader) error { - ret := _m.Called(backup, restore, results) +// PutBackupItemOperations provides a mock function with given fields: backup, backupItemOperations +func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperations io.Reader) error { + ret := _m.Called(backup, backupItemOperations) var r0 error - if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { - r0 = rf(backup, restore, results) + if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { + r0 = rf(backup, backupItemOperations) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// PutBackupMetadata provides a mock function with given fields: backup, backupMetadata +func (_m *BackupStore) PutBackupMetadata(backup string, backupMetadata io.Reader) error { + ret := _m.Called(backup, backupMetadata) + + var r0 error + if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { + r0 = rf(backup, backupMetadata) } else { r0 = ret.Error(0) } @@ -289,13 +421,13 @@ func (_m *BackupStore) PutRestoreItemOperations(backup string, restore string, r return r0 } -// PutBackupItemOperations provides a mock function with given fields: backup, backupItemOperations -func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperations io.Reader) error { - ret := _m.Called(backup, backupItemOperations) +// PutRestoreLog provides a mock function with given fields: backup, restore, log +func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error { + ret := _m.Called(backup, restore, log) var r0 error - if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { - r0 = rf(backup, backupItemOperations) + if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { + r0 = rf(backup, restore, log) } else { r0 = ret.Error(0) } @@ -303,29 +435,33 @@ func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperatio return r0 } -func (_m *BackupStore) GetCSIVolumeSnapshots(backup string) ([]*snapshotv1api.VolumeSnapshot, error) { - panic("Not implemented") - return nil, nil -} +// PutRestoreResults provides a mock function with given fields: backup, restore, results +func (_m *BackupStore) PutRestoreResults(backup string, restore string, results io.Reader) error { + ret := _m.Called(backup, restore, results) -func (_m *BackupStore) GetCSIVolumeSnapshotContents(backup string) ([]*snapshotv1api.VolumeSnapshotContent, error) { - panic("Not implemented") - return nil, nil -} + var r0 error + if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { + r0 = rf(backup, restore, results) + } else { + r0 = ret.Error(0) + } -func (_m *BackupStore) GetCSIVolumeSnapshotClasses(backup string) ([]*snapshotv1api.VolumeSnapshotClass, error) { - panic("Not implemented") - return nil, nil + return r0 } -func (_m *BackupStore) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) { - panic("implement me") - return nil, nil +type mockConstructorTestingTNewBackupStore interface { + mock.TestingT + Cleanup(func()) } -func (_m *BackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.RestoreOperation, error) { - panic("implement me") - return nil, nil +// NewBackupStore creates a new instance of BackupStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewBackupStore(t mockConstructorTestingTNewBackupStore) *BackupStore { + mock := &BackupStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } func (_m *BackupStore) PutRestoredResourceList(restore string, results io.Reader) error { diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 82d0f5ddb2..2e6c8edab2 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -61,7 +61,9 @@ type BackupStore interface { ListBackups() ([]string, error) PutBackup(info BackupInfo) error + PutBackupMetadata(backup string, backupMetadata io.Reader) error PutBackupItemOperations(backup string, backupItemOperations io.Reader) error + PutBackupContents(backup string, backupContents io.Reader) error GetBackupMetadata(name string) (*velerov1api.Backup, error) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) @@ -315,6 +317,10 @@ func (s *objectBackupStore) GetBackupMetadata(name string) (*velerov1api.Backup, return backupObj, nil } +func (s *objectBackupStore) PutBackupMetadata(backup string, backupMetadata io.Reader) error { + return seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupMetadataKey(backup), backupMetadata) +} + func (s *objectBackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) { // if the volumesnapshots file doesn't exist, we don't want to return an error, since // a legacy backup or a backup with no snapshots would not have this file, so check for @@ -550,6 +556,10 @@ func (s *objectBackupStore) PutBackupItemOperations(backup string, backupItemOpe return seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupItemOperationsKey(backup), backupItemOperations) } +func (s *objectBackupStore) PutBackupContents(backup string, backupContents io.Reader) error { + return seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupContentsKey(backup), backupContents) +} + func (s *objectBackupStore) GetDownloadURL(target velerov1api.DownloadTarget) (string, error) { switch target.Kind { case velerov1api.DownloadTargetKindBackupContents: diff --git a/pkg/persistence/object_store_test.go b/pkg/persistence/object_store_test.go index 04301b6f13..5a12f06fcf 100644 --- a/pkg/persistence/object_store_test.go +++ b/pkg/persistence/object_store_test.go @@ -37,6 +37,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" providermocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" @@ -461,14 +462,22 @@ func TestGetBackupItemOperations(t *testing.T) { operations := []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ - BackupName: "test-backup", - ResourceIdentifier: "item-1", + BackupName: "test-backup", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns", + Name: "item-1", + }, }, }, { Spec: itemoperation.BackupOperationSpec{ - BackupName: "test-backup", - ResourceIdentifier: "item-2", + BackupName: "test-backup", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns", + Name: "item-2", + }, }, }, } diff --git a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go index b3739796f7..c3121b1aac 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go @@ -96,6 +96,11 @@ func (r *RestartableBackupItemAction) getDelegate() (biav2.BackupItemAction, err return r.getBackupItemAction() } +// Name returns the plugin's name. +func (r *RestartableBackupItemAction) Name() string { + return r.Key.Name +} + // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r *RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { delegate, err := r.getDelegate() @@ -107,10 +112,10 @@ func (r *RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, erro } // Execute restarts the plugin's process if needed, then delegates the call. -func (r *RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (r *RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { delegate, err := r.getDelegate() if err != nil { - return nil, nil, "", err + return nil, nil, "", nil, err } return delegate.Execute(item, backup) @@ -148,15 +153,20 @@ func NewAdaptedV1RestartableBackupItemAction(v1Restartable *biav1cli.Restartable return r } +// Name restarts the plugin's name. +func (r *AdaptedV1RestartableBackupItemAction) Name() string { + return r.V1Restartable.Key.Name +} + // AppliesTo delegates to the v1 AppliesTo call. func (r *AdaptedV1RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { return r.V1Restartable.AppliesTo() } // Execute delegates to the v1 Execute call, returning an empty operationID. -func (r *AdaptedV1RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (r *AdaptedV1RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { updatedItem, additionalItems, err := r.V1Restartable.Execute(item, backup) - return updatedItem, additionalItems, "", err + return updatedItem, additionalItems, "", nil, err } // Progress returns with an error since v1 plugins will never return an operationID, which means that diff --git a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go index 24e3c12a60..9d2f66c34b 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go @@ -145,8 +145,8 @@ func TestRestartableBackupItemActionDelegatedFunctions(t *testing.T) { restartabletest.RestartableDelegateTest{ Function: "Execute", Inputs: []interface{}{pv, b}, - ExpectedErrorOutputs: []interface{}{nil, ([]velero.ResourceIdentifier)(nil), "", errors.Errorf("reset error")}, - ExpectedDelegateOutputs: []interface{}{pvToReturn, additionalItems, "", errors.Errorf("delegate error")}, + ExpectedErrorOutputs: []interface{}{nil, ([]velero.ResourceIdentifier)(nil), "", ([]velero.ResourceIdentifier)(nil), errors.Errorf("reset error")}, + ExpectedDelegateOutputs: []interface{}{pvToReturn, additionalItems, "", ([]velero.ResourceIdentifier)(nil), errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Progress", diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go index 925cf037c7..b02c35f3c4 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go @@ -76,15 +76,15 @@ func (c *BackupItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error }, nil } -func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { itemJSON, err := json.Marshal(item.UnstructuredContent()) if err != nil { - return nil, nil, "", errors.WithStack(err) + return nil, nil, "", nil, errors.WithStack(err) } backupJSON, err := json.Marshal(backup) if err != nil { - return nil, nil, "", errors.WithStack(err) + return nil, nil, "", nil, errors.WithStack(err) } req := &protobiav2.ExecuteRequest{ @@ -95,12 +95,12 @@ func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup * res, err := c.grpcClient.Execute(context.Background(), req) if err != nil { - return nil, nil, "", common.FromGRPCError(err) + return nil, nil, "", nil, common.FromGRPCError(err) } var updatedItem unstructured.Unstructured if err := json.Unmarshal(res.Item, &updatedItem); err != nil { - return nil, nil, "", errors.WithStack(err) + return nil, nil, "", nil, errors.WithStack(err) } var additionalItems []velero.ResourceIdentifier @@ -118,7 +118,22 @@ func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup * additionalItems = append(additionalItems, newItem) } - return &updatedItem, additionalItems, res.OperationID, nil + var itemsToUpdate []velero.ResourceIdentifier + + for _, itm := range res.ItemsToUpdate { + newItem := velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: itm.Group, + Resource: itm.Resource, + }, + Namespace: itm.Namespace, + Name: itm.Name, + } + + itemsToUpdate = append(itemsToUpdate, newItem) + } + + return &updatedItem, additionalItems, res.OperationID, itemsToUpdate, nil } func (c *BackupItemActionGRPCClient) Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) { @@ -167,3 +182,9 @@ func (c *BackupItemActionGRPCClient) Cancel(operationID string, backup *api.Back return nil } + +// This shouldn't be called on the GRPC client since the RestartableBackupItemAction won't delegate +// this method +func (c *BackupItemActionGRPCClient) Name() string { + return "" +} diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go index 2de33ed7c4..246214e0ac 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go @@ -107,7 +107,7 @@ func (s *BackupItemActionGRPCServer) Execute( return nil, common.NewGRPCError(errors.WithStack(err)) } - updatedItem, additionalItems, operationID, err := impl.Execute(&item, &backup) + updatedItem, additionalItems, operationID, itemsToUpdate, err := impl.Execute(&item, &backup) if err != nil { return nil, common.NewGRPCError(err) } @@ -132,6 +132,9 @@ func (s *BackupItemActionGRPCServer) Execute( for _, item := range additionalItems { res.AdditionalItems = append(res.AdditionalItems, backupResourceIdentifierToProto(item)) } + for _, item := range itemsToUpdate { + res.ItemsToUpdate = append(res.ItemsToUpdate, backupResourceIdentifierToProto(item)) + } return res, nil } @@ -210,3 +213,9 @@ func backupResourceIdentifierToProto(id velero.ResourceIdentifier) *proto.Resour Name: id.Name, } } + +// This shouldn't be called on the GRPC server since the server won't ever receive this request, as +// the RestartableBackupItemAction in Velero won't delegate this to the server +func (c *BackupItemActionGRPCServer) Name() string { + return "" +} diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go index d9e2d2cea1..a0c91d4e16 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go @@ -97,6 +97,7 @@ func TestBackupItemActionGRPCServerExecute(t *testing.T) { implUpdatedItem runtime.Unstructured implAdditionalItems []velero.ResourceIdentifier implOperationID string + implItemsToUpdate []velero.ResourceIdentifier implError error expectError bool skipMock bool @@ -153,7 +154,7 @@ func TestBackupItemActionGRPCServerExecute(t *testing.T) { defer itemAction.AssertExpectations(t) if !test.skipMock { - itemAction.On("Execute", &validItemObject, &validBackupObject).Return(test.implUpdatedItem, test.implAdditionalItems, test.implOperationID, test.implError) + itemAction.On("Execute", &validItemObject, &validBackupObject).Return(test.implUpdatedItem, test.implAdditionalItems, test.implOperationID, test.implItemsToUpdate, test.implError) } s := &BackupItemActionGRPCServer{mux: &common.ServerMux{ diff --git a/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go b/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go index bbcaa50e6d..66c08e1c86 100644 --- a/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go +++ b/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go @@ -102,6 +102,7 @@ type ExecuteResponse struct { Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` OperationID string `protobuf:"bytes,3,opt,name=operationID,proto3" json:"operationID,omitempty"` + ItemsToUpdate []*generated.ResourceIdentifier `protobuf:"bytes,4,rep,name=itemsToUpdate,proto3" json:"itemsToUpdate,omitempty"` } func (x *ExecuteResponse) Reset() { @@ -157,6 +158,13 @@ func (x *ExecuteResponse) GetOperationID() string { return "" } +func (x *ExecuteResponse) GetItemsToUpdate() []*generated.ResourceIdentifier { + if x != nil { + return x.ItemsToUpdate + } + return nil +} + type BackupItemActionAppliesToRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -438,7 +446,7 @@ var file_backupitemaction_v2_BackupItemAction_proto_rawDesc = []byte{ 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x22, 0x90, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x22, 0xd5, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, @@ -447,63 +455,67 @@ var file_backupitemaction_v2_BackupItemAction_proto_rawDesc = []byte{ 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, - 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, - 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x22, 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x73, - 0x0a, 0x1f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, - 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x22, 0x5c, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, + 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x43, 0x0a, 0x0d, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x54, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0d, 0x69, 0x74, 0x65, 0x6d, + 0x73, 0x54, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, + 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, + 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, + 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, + 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, + 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x22, 0x73, 0x0a, 0x1f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, - 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x22, 0x71, 0x0a, 0x1d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, - 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x32, 0xbc, 0x02, 0x0a, 0x10, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, - 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x09, 0x41, 0x70, 0x70, - 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, - 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x76, - 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x12, - 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, - 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, + 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, + 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x5c, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, + 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, + 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x71, 0x0a, 0x1d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, + 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x32, 0xbc, 0x02, 0x0a, 0x10, 0x42, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x58, + 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x24, 0x2e, 0x76, 0x32, + 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, + 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x65, 0x12, 0x12, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x08, + 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, - 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x21, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, - 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, - 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, + 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x21, 0x2e, + 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, + 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, + 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, + 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -534,21 +546,22 @@ var file_backupitemaction_v2_BackupItemAction_proto_goTypes = []interface{}{ } var file_backupitemaction_v2_BackupItemAction_proto_depIdxs = []int32{ 7, // 0: v2.ExecuteResponse.additionalItems:type_name -> generated.ResourceIdentifier - 8, // 1: v2.BackupItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector - 9, // 2: v2.BackupItemActionProgressResponse.progress:type_name -> generated.OperationProgress - 2, // 3: v2.BackupItemAction.AppliesTo:input_type -> v2.BackupItemActionAppliesToRequest - 0, // 4: v2.BackupItemAction.Execute:input_type -> v2.ExecuteRequest - 4, // 5: v2.BackupItemAction.Progress:input_type -> v2.BackupItemActionProgressRequest - 6, // 6: v2.BackupItemAction.Cancel:input_type -> v2.BackupItemActionCancelRequest - 3, // 7: v2.BackupItemAction.AppliesTo:output_type -> v2.BackupItemActionAppliesToResponse - 1, // 8: v2.BackupItemAction.Execute:output_type -> v2.ExecuteResponse - 5, // 9: v2.BackupItemAction.Progress:output_type -> v2.BackupItemActionProgressResponse - 10, // 10: v2.BackupItemAction.Cancel:output_type -> google.protobuf.Empty - 7, // [7:11] is the sub-list for method output_type - 3, // [3:7] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 7, // 1: v2.ExecuteResponse.itemsToUpdate:type_name -> generated.ResourceIdentifier + 8, // 2: v2.BackupItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector + 9, // 3: v2.BackupItemActionProgressResponse.progress:type_name -> generated.OperationProgress + 2, // 4: v2.BackupItemAction.AppliesTo:input_type -> v2.BackupItemActionAppliesToRequest + 0, // 5: v2.BackupItemAction.Execute:input_type -> v2.ExecuteRequest + 4, // 6: v2.BackupItemAction.Progress:input_type -> v2.BackupItemActionProgressRequest + 6, // 7: v2.BackupItemAction.Cancel:input_type -> v2.BackupItemActionCancelRequest + 3, // 8: v2.BackupItemAction.AppliesTo:output_type -> v2.BackupItemActionAppliesToResponse + 1, // 9: v2.BackupItemAction.Execute:output_type -> v2.ExecuteResponse + 5, // 10: v2.BackupItemAction.Progress:output_type -> v2.BackupItemActionProgressResponse + 10, // 11: v2.BackupItemAction.Cancel:output_type -> google.protobuf.Empty + 8, // [8:12] is the sub-list for method output_type + 4, // [4:8] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_backupitemaction_v2_BackupItemAction_proto_init() } diff --git a/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto b/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto index 5f7bfe7d33..366141dcad 100644 --- a/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto +++ b/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto @@ -16,6 +16,7 @@ message ExecuteResponse { bytes item = 1; repeated generated.ResourceIdentifier additionalItems = 2; string operationID = 3; + repeated generated.ResourceIdentifier itemsToUpdate = 4; } service BackupItemAction { diff --git a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go index a4a70234e7..920f07965e 100644 --- a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go +++ b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go @@ -29,6 +29,12 @@ import ( // BackupItemAction is an actor that performs an operation on an individual item being backed up. type BackupItemAction interface { + // Name returns the name of this BIA. Plugins which implement this interface must defined Name, + // but its content is unimportant, as it won't actually be called via RPC. Velero's plugin infrastructure + // will implement this directly rather than delegating to the RPC plugin in order to return the name + // that the plugin was registered under. The plugins must implement the method to complete the interface. + Name() string + // AppliesTo returns information about which resources this action should be invoked for. // A BackupItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. @@ -37,8 +43,13 @@ type BackupItemAction interface { // Execute allows the BackupItemAction to perform arbitrary logic with the item being backed up, // including mutating the item itself prior to backup. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying - // additional related items that should be backed up. - Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) + // additional related items that should be backed up now, an optional operationID for actions which + // initiate asynchronous actions, and a second slice of ResourceIdentifiers specifying related items + // which should be backed up after all asynchronous operations have completed. This last field will be + // ignored if operationID is empty, and should not be filled in unless the resource must be updated in the + // backup after async operations complete (i.e. some of the item's kubernetes metadata will be updated + // during the asynch operation which will be required during restore) + Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) // Progress allows the BackupItemAction to report on progress of an asynchronous action. // For the passed-in operation, the plugin will return an OperationProgress struct, indicating diff --git a/pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go b/pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go index 63be81d4d6..4264424778 100644 --- a/pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go +++ b/pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery v2.16.0. DO NOT EDIT. package v2 @@ -67,7 +67,7 @@ func (_m *BackupItemAction) Cancel(operationID string, backup *v1.Backup) error } // Execute provides a mock function with given fields: item, backup -func (_m *BackupItemAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (_m *BackupItemAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { ret := _m.Called(item, backup) var r0 runtime.Unstructured @@ -95,14 +95,37 @@ func (_m *BackupItemAction) Execute(item runtime.Unstructured, backup *v1.Backup r2 = ret.Get(2).(string) } - var r3 error - if rf, ok := ret.Get(3).(func(runtime.Unstructured, *v1.Backup) error); ok { + var r3 []velero.ResourceIdentifier + if rf, ok := ret.Get(3).(func(runtime.Unstructured, *v1.Backup) []velero.ResourceIdentifier); ok { r3 = rf(item, backup) } else { - r3 = ret.Error(3) + if ret.Get(3) != nil { + r3 = ret.Get(3).([]velero.ResourceIdentifier) + } } - return r0, r1, r2, r3 + var r4 error + if rf, ok := ret.Get(4).(func(runtime.Unstructured, *v1.Backup) error); ok { + r4 = rf(item, backup) + } else { + r4 = ret.Error(4) + } + + return r0, r1, r2, r3, r4 +} + +// Name provides a mock function with given fields: +func (_m *BackupItemAction) Name() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 } // Progress provides a mock function with given fields: operationID, backup @@ -125,3 +148,18 @@ func (_m *BackupItemAction) Progress(operationID string, backup *v1.Backup) (vel return r0, r1 } + +type mockConstructorTestingTNewBackupItemAction interface { + mock.TestingT + Cleanup(func()) +} + +// NewBackupItemAction creates a new instance of BackupItemAction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewBackupItemAction(t mockConstructorTestingTNewBackupItemAction) *BackupItemAction { + mock := &BackupItemAction{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/plugin/velero/shared.go b/pkg/plugin/velero/shared.go index 6316a1e0bf..636243eb75 100644 --- a/pkg/plugin/velero/shared.go +++ b/pkg/plugin/velero/shared.go @@ -68,6 +68,20 @@ type ResourceIdentifier struct { Name string } +func (in *ResourceIdentifier) DeepCopy() *ResourceIdentifier { + if in == nil { + return nil + } + out := new(ResourceIdentifier) + in.DeepCopyInto(out) + return out +} + +func (in *ResourceIdentifier) DeepCopyInto(out *ResourceIdentifier) { + *out = *in + out.GroupResource = in.GroupResource +} + // OperationProgress describes progress of an asynchronous plugin operation. type OperationProgress struct { // True when the operation has completed, either successfully or with a failure diff --git a/site/content/docs/main/api-types/backup.md b/site/content/docs/main/api-types/backup.md index 8c703404db..ea9447b9ad 100644 --- a/site/content/docs/main/api-types/backup.md +++ b/site/content/docs/main/api-types/backup.md @@ -33,6 +33,10 @@ spec: # CSI VolumeSnapshot status turns to ReadyToUse during creation, before # returning error as timeout. The default value is 10 minute. csiSnapshotTimeout: 10m + # ItemOperationTimeout specifies the time used to wait for + # asynchronous BackupItemAction operations + # The default value is 1 hour. + csiSnapshotTimeout: 1h # Array of namespaces to include in the backup. If unspecified, all namespaces are included. # Optional. includedNamespaces: @@ -146,7 +150,8 @@ status: expiration: null # The current phase. # Valid values are New, FailedValidation, InProgress, WaitingForPluginOperations, - # WaitingForPluginOperationsPartiallyFailed, Completed, PartiallyFailed, Failed. + # WaitingForPluginOperationsPartiallyFailed, FinalizingafterPluginOperations, + # FinalizingAfterPluginOperationsPartiallyFailed, Completed, PartiallyFailed, Failed. phase: "" # An array of any validation errors encountered. validationErrors: null @@ -158,6 +163,12 @@ status: volumeSnapshotsAttempted: 2 # Number of volume snapshots that Velero successfully created for this backup. volumeSnapshotsCompleted: 1 + # Number of attempted async BackupItemAction operations for this backup. + asyncBackupItemOperationsAttempted: 2 + # Number of async BackupItemAction operations that Velero successfully completed for this backup. + asyncBackupItemOperationsCompleted: 1 + # Number of async BackupItemAction operations that ended in failure for this backup. + asyncBackupItemOperationsFailed: 0 # Number of warnings that were logged by the backup. warnings: 2 # Number of errors that were logged by the backup.