From ecbafce8f46136475508397c48e16bbec9d68de4 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 24 Jan 2023 12:59:53 -0500 Subject: [PATCH] BIAv2 async operations controller work Signed-off-by: Scott Seago --- changelogs/unreleased/5849-sseago | 1 + config/crd/v1/bases/velero.io_backups.yaml | 21 + config/crd/v1/bases/velero.io_schedules.yaml | 5 + config/crd/v1/crds/crds.go | 4 +- pkg/apis/velero/v1/backup.go | 39 +- pkg/apis/velero/v1/zz_generated.deepcopy.go | 1 + pkg/backup/backup_test.go | 236 +++++++- pkg/backup/item_backupper.go | 32 +- pkg/backup/request.go | 11 + pkg/builder/backup_builder.go | 6 + pkg/cmd/cli/backup/create.go | 5 +- pkg/cmd/cli/backup/create_test.go | 2 + pkg/cmd/cli/backup/logs.go | 4 +- pkg/cmd/cli/schedule/create.go | 1 + pkg/cmd/server/server.go | 42 +- pkg/cmd/server/server_test.go | 12 +- pkg/cmd/util/output/backup_describer.go | 63 +++ .../async_backup_operations_controller.go | 514 ++++++++++++++++++ ...async_backup_operations_controller_test.go | 308 +++++++++++ pkg/controller/backup_controller.go | 151 +++-- pkg/controller/backup_sync_controller.go | 20 + pkg/controller/backup_sync_controller_test.go | 188 +++++-- pkg/controller/constants.go | 2 + pkg/controller/download_request_controller.go | 22 +- .../download_request_controller_test.go | 1 + pkg/itemoperation/backup_operation.go | 45 +- pkg/itemoperation/restore_operation.go | 6 +- pkg/itemoperation/shared.go | 38 +- pkg/persistence/mocks/backup_store.go | 194 +++++-- pkg/persistence/object_store.go | 5 + pkg/persistence/object_store_test.go | 17 +- .../v2/restartable_backup_item_action.go | 18 +- .../v2/restartable_backup_item_action_test.go | 4 +- .../v2/backup_item_action_client.go | 33 +- .../v2/backup_item_action_server.go | 11 +- .../v2/backup_item_action_test.go | 3 +- .../v2/BackupItemAction.pb.go | 155 +++--- .../v2/BackupItemAction.proto | 1 + .../backupitemaction/v2/backup_item_action.go | 12 +- .../backupitemaction/v2/BackupItemAction.go | 50 +- pkg/plugin/velero/shared.go | 14 + 41 files changed, 2010 insertions(+), 287 deletions(-) create mode 100644 changelogs/unreleased/5849-sseago create mode 100644 pkg/controller/async_backup_operations_controller.go create mode 100644 pkg/controller/async_backup_operations_controller_test.go diff --git a/changelogs/unreleased/5849-sseago b/changelogs/unreleased/5849-sseago new file mode 100644 index 00000000000..4e64ab652f9 --- /dev/null +++ b/changelogs/unreleased/5849-sseago @@ -0,0 +1 @@ +BIAv2 async operations controller work diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 2fb76533a16..1cfa6931452 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -273,6 +273,11 @@ spec: type: string nullable: true type: array + itemOperationTimeout: + description: ItemOperationTimeout specifies the time used to wait + for asynchronous BackupItemAction operations The default value is + 1 hour. + type: string labelSelector: description: LabelSelector is a metav1.LabelSelector to filter with when adding individual objects to the backup. If empty or nil, all @@ -415,6 +420,20 @@ spec: status: description: BackupStatus captures the current status of a Velero backup. properties: + asyncBackupItemOperationsAttempted: + description: AsyncBackupItemOperationsAttempted is the total number + of attempted async BackupItemAction operations for this backup. + type: integer + asyncBackupItemOperationsCompleted: + description: AsyncBackupItemOperationsCompleted is the total number + of successfully completed async BackupItemAction operations for + this backup. + type: integer + asyncBackupItemOperationsFailed: + description: AsyncBackupItemOperationsFailed is the total number of + async BackupItemAction operations for this backup which ended with + an error. + type: integer completionTimestamp: description: CompletionTimestamp records the time a backup was completed. Completion time is recorded even on failed backups. Completion time @@ -457,6 +476,8 @@ spec: - InProgress - WaitingForPluginOperations - WaitingForPluginOperationsPartiallyFailed + - FinalizingAfterPluginOperations + - FinalizingAfterPluginOperationsPartiallyFailed - Completed - PartiallyFailed - Failed diff --git a/config/crd/v1/bases/velero.io_schedules.yaml b/config/crd/v1/bases/velero.io_schedules.yaml index 9e7454d5177..22405ad12d5 100644 --- a/config/crd/v1/bases/velero.io_schedules.yaml +++ b/config/crd/v1/bases/velero.io_schedules.yaml @@ -308,6 +308,11 @@ spec: type: string nullable: true type: array + itemOperationTimeout: + description: ItemOperationTimeout specifies the time used to wait + for asynchronous BackupItemAction operations The default value + is 1 hour. + type: string labelSelector: description: LabelSelector is a metav1.LabelSelector to filter with when adding individual objects to the backup. If empty diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 01197466248..bb669942a13 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -30,14 +30,14 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WAo\xdc6\x13\xbd\xebW\f\xf2\x1dr\xf9\xa4M\xd0C\v\xddR\xb7\x05\x82&\x86a\a\xbe\x14=P\xe4\xec.c\x8ad\xc9\xe1\xa6ۢ\xff\xbd\x18R\xf2j%\xd9\x1b\a\xa8n\"\x87of\xde\xcc\x1bQU]ו\xf0\xfa\x1eC\xd4ζ \xbc\xc6?\t-\xbf\xc5\xe6\xe1\x87\xd8h\xb79\xbc\xad\x1e\xb4U-\\\xa5H\xae\xbf\xc5\xe8R\x90\xf8\x13n\xb5դ\x9d\xadz$\xa1\x04\x89\xb6\x02\x10\xd6:\x12\xbc\x1c\xf9\x15@:K\xc1\x19\x83\xa1ޡm\x1eR\x87]\xd2Fa\xc8\xe0\xa3\xebÛ\xe6\xfb\xe6M\x05 \x03\xe6\xe3\x9ft\x8f\x91D\xef[\xb0ɘ\n\xc0\x8a\x1e[\xe8\x84|H>\xa0wQ\x93\v\x1acs@\x83\xc15\xdaUѣd\xb7\xbb\xe0\x92o\xe1\xb4QN\x0f!\x95t~\xcc@\xb7#\xd01o\x19\x1d\xe9\xd7\xd5\xed\x0f:R6\xf1&\x05a\xd6\x02\xc9\xdbQ\xdb]2\",\f\xd8A\x94\xcec\v\xd7\x1c\x8b\x17\x12U\x050P\x90c\xabA(\x95I\x15\xe6&hK\x18\xae\x9cI\xfdHf\r\x9f\xa3\xb37\x82\xf6-4#\xed͂\xb2l;\x12\xf6n\x87\xc3;\x1dٹ\x12\x84K0f\xae9\xc5\xfa\xe9\xe8\xf1\f\xe5D\x04L\xf6\nb\xa4\xa0\xed\xae:\x19\x1f\xde\x16*\xe4\x1e{\xd1\x0e\xb6Σ}w\xf3\xfe\xfe\xbb\xbb\xb3e\x00\x1f\x9c\xc7@z,Oy&}9Y\x05P\x18eОr\u05fcf\xc0b\x05\x8a\x1b\x12#\xd0\x1eGNQ\r1\x80\xdb\x02\xedu\x84\x80>`D[Z\xf4\f\x18\xd8HXp\xddg\x94\xd4\xc0\x1d\x06\x86\x81\xb8w\xc9(\xee\xe3\x03\x06\x82\x80\xd2\xed\xac\xfe\xeb\x11;\x02\xb9\xec\xd4\b¡GNO\xae\xa1\x15\x06\x0e\xc2$\xfc?\b\xab\xa0\x17G\b\xc8^ \xd9\t^6\x89\r|t\x01AۭkaO\xe4c\xbb\xd9\xec4\x8dz\x94\xae\xef\x93\xd5t\xdcdi\xe9.\x91\vq\xa3\xf0\x80f\x13\xf5\xae\x16A\xee5\xa1\xa4\x14p#\xbc\xaes\xe86k\xb2\xe9\xd5\xff\u00a0\xe0\xf8\xfa,\xd6E-˓\xc5\xf2L\x05X-\xa0#\x88\xe1h\xc9\xe2D4/1;\xb7?\xdf}\x82\xd1u.Ɯ\xfd\xcc\xfb\xe9`<\x95\x80\t\xd3v\x8b\xa1\x14q\x1b\\\x9f1\xd1*ﴥ\xfc\"\x8dF;\xa7?\xa6\xae\xd7\xc4u\xff#a$\xaeU\x03WyHA\x87\x90<\xabA5\xf0\xde\u0095\xe8\xd1\\\x89\x88\xffy\x01\x98\xe9X3\xb1_W\x82\xe9|\x9d\x1b\x17\xd6&\x1b\xe3\b|\xa2^\xf3\xb1v\xe7Qr\xf9\x98A>\xaa\xb7Zfm\xc0\xd6\x05\x10\v\xfb\xe6\fz]\xba\xfc\x94\xe1wG.\x88\x1d~p\x05sn\xb4\x1a\xdb\xec\xcc\x18\x1cO\x96\"c\\7\\`\x03\xd0^\xd0D\xbf$\xb4}\x1c\x03\xab\xf90\x92-S\bhi\x80\xc97\x88o\x1e\x99FD\x9a\x8c\v\xbe\xcd]\xe8\x80\x0f\xcb\x13c`\f\x06\xc4\v\xd3\xf9\xf2E̿\xba\xb9hk\x93e\xebB/\xa8\\\x17k\x06ZX\xf0\xb5\\t\x06[\xa0\x90\x96\xdb\xcf\xcdQ\x8cQ\xec.e\xf7\xb1X\x95\xcb\xc5p\x04D\xe7\x12=A=\xed\x97Q\xc0\x85r\\\x88\xd4\xefE\xbc\x14\xe7\r۬5\xc4\xec{\xf5\\\bO\xcd\xcck\xfc\xb2\xb2z\x8bB-u\\õ\xa3\xf5\xad'3\\U\xc5b1\xf2=LM\xea\x1c\x8b\x90\xa7+\xa9{\xbcW\xb6\xf0\xf7?\xd5IXBJ\xf4\x84\xeaz\xfe\a6\xcc\xf7\xf1\x87*\xbfJg\xcb\x0fPl\xe1\xb7߫\xe2\n\xd5\xfd\xf8\x93ċ\xff\x06\x00\x00\xff\xff\xc8p\x98۸\x0e\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Ms\xdc:rw\xfd\x8a.\xe5\xe0\xdd*\xcd\xe8\xb9rHJ7?Y\xae\xa8ދ\xad\xb2\xb4\xdaC\x92\x03\x86\xec\x99\xc1\x13\bp\x01p\xe4I*\xff=\xd5\x00\xf8\r\x92\x18Y\xda}/e\\l\x91@\x03\xe8n\xf4\x17\x9a=g\xab\xd5ꌕ\xfc\x11\xb5\xe1J^\x01+9~\xb3(\xe9/\xb3~\xfaW\xb3\xe6\xea\xf2\xf0\xfe\xec\x89\xcb\xfc\n\xae+cU\xf1\x15\x8d\xaat\x86\x1fq\xcb%\xb7\\ɳ\x02-˙eWg\x00LJe\x19=6\xf4'@\xa6\xa4\xd5J\bԫ\x1d\xca\xf5S\xb5\xc1M\xc5E\x8e\xda\x01\xaf\xa7>\xfc\xb4\xfe\x97\xf5Og\x00\x99F7\xfc\x81\x17h,+\xca+\x90\x95\x10g\x00\x92\x15x\x05\x1b\x96=U\xa5Y\x1fP\xa0Vk\xae\xceL\x89\x19͵Ӫ*\xaf\xa0}ᇄu\xf8=\xfc\xecF\xbb\a\x82\x1b\xfbK\xe7\xe1\xaf\xdcX\xf7\xa2\x14\x95f\xa2\x99\xc9=3\\\xee*\xc1t\xfd\xf4\f\xc0d\xaa\xc4+\xf8LS\x94,\xc3\xfc\f l\xc7M\xb9\n\v>\xbc\xf7\x10\xb2=\x16̯\x05@\x95(?\xdc\xdd>\xfe\xf3}\xef1@\x8e&Ӽ\xb4\x0e)~a\xc0\r0xt\xdb\x02\x1d\xd0\x0fv\xcf,h,5\x1a\x94ր\xdd#d\xac\xb4\x95FP[\xf8\xa5ڠ\x96h\xd14\xa0\x012Q\x19\x8b\x1a\x8ce\x16\x81Y`P*.-p\t\x96\x17\b\u007f\xfapw\vj\xf3\x1bf\xd6\x00\x9390cTƙ\xc5\x1c\x0eJT\x05\xfa\xb1\u007f^7PK\xadJԖ\xd7x\xf6\xad\xc3U\x9d\xa7\x83\xed\xbd#\f\xf8^\x90\x13;\xa1\xdfF\xc0\"\xe6\x01i\xb4\x1f\xbb\xe7\xa6ݮ\xe3\x90\x1e`\xa0NL\x86ů\xe1\x1e5\x81\x01\xb3W\x95ȉ\v\x0f\xa8\ta\x99\xdaI\xfe\xdf\rl\x03V\xb9I\x05\xb3\x18\x18\xa0m\\ZԒ\t80Q\xe1\x85CI\xc1\x8e\xa0\x91f\x81Jv\xe0\xb9.f\r\xff\xae4\x02\x97[u\x05{kKsuy\xb9\xe3\xb6>M\x99*\x8aJr{\xbct\a\x83o*\xab\xb4\xb9\xcc\xf1\x80\xe2\xd2\xf0݊\xe9l\xcf-fD\xc8KV\xf2\x95[\xbat'j]\xe4\xffT3\x80y\xd7[\xab=\x123\x1a\xab\xb9\xdcu^8\xae\x9f\xa1\x00\x1d\x00\xcf_~\xa8\xdfE\x8bhzD\xd8\xf9zs\xff\xd0\xe5=n\x86\xd8wx\xef0dK\x02B\x18\x97[Ԟ\x88[\xad\n\a\x13e\xee\xb9ϱ\xae\xe0(\x87\xe87զ\xe0\x96\xe8\xfe\xb7\n\r1\xb9Zõ\x131\xb0A\xa8ʜ8s\r\xb7\x12\xaeY\x81\xe2\x9a\x19|s\x02\x10\xa6͊\x10\x9bF\x82\xaet\x1cv\xf6X뼨e\xd9\x04\xbd\xbc@\xb8/1\xeb\x1d\x18\x1aŷ\xd8V$K\xc6x\x06 ^\x9e\xe4\x01.\x8dE\x96\xaf\xcf_\x93@\xf8-\x13U\x8eyc\xb6\x8cd\xc0\x8087\xa3\x01Τc\\\x92\xd6 #\x8a\x90+۷d\x98D\xb6\xca4\x02\xc9m.=\x97\xf3\xeeGZ.͋3h\xfa\x192\x13\"\xfa\xd4+\xa3\xf4D\xe1\xf4\x1c\x99\xf9\xa4\x96S2c\x86y/\x93@\x97\xf3aR<Džܗ\x17d\xbc$f;~\xf7\xc5XJNˋ2Y\x16\x13\x02\x13\xf3W\xfa\x99)\xf3 O\xc8ZIB\xcer\x86\xca\xc9y)!\x0fdv\x1f\xc9\xd9(\x91<\x93Y\xc0\x939(s\xd9%\vQ\xa9q\xe6IzN\xc9,h\x97o\xb2\x9cI\xf2z\xf9\xa2\xafa\x03O\x8b\x9a\xc5l\x90E\x1by~}\x8b\xf9\x1e\xa7dy,b\xec\x85\x19\x1dM\xc6\xc6ļ\xa7\xe6q\xf4\xf34&\x80\xa6doLdgL@\x9c\xcd\xd9H\xcdɘ\x80\xbd\xa0vg\xb9d\xe6e\xfcC`X\xd4o\xe2\xef\xc5Q/ݘ\xd2=sq\xc9B\xff2\xe8N\xb4\xac\xad\xa6y\xf33fyr\xbb?\xdd\xfc,*ay)\\8\xff\xc0\xf3\xa8\xd3h\xf7xl>\xeb\xfcM\xb9Ϝ6G\a\xe9\xcb׆=\xd7\x03#\x9a\x19xF!\x80Řk\xb4\xf3\xcc\u007f˞\xa9\x15\x92̧\x03\x17>X\r\x9f\xbc_x\x0ev_r\xc5\"\x9ev\x8f\x05A\xa9\xbf|=\xc1\xfd\x987\x10\xbd-\xeb\x9e\xfd\xadB}\x04u@\xddZ\f\v\xdf\x11\xf8\x83f*\xd1&n\x05\xf9\xe1+(\f\f\xe7\xf6\xc0\xc1\a\xe9UX\x14\xec`\x8d\x0e\x0e\x9dy\xd1К\xc4\x1b\xf9\x01\x13]\xe3\x81\x0fՌ\x8e\xbc_\xb2=S\x93\xf0\xdf\xd6u8\xddyXT\xdbo\xe2@\xbc܅\x98\x01\x99\x9aT\x9fv\x01\xb5\x98D\xffV\xaeĒ3\x91lE\xa5%ɿEr\xfc\tI\xf1'8\x15\xa7\xb9\x15\xc9hJI~\u007f\x13\xe7\xe2\r\u074b\xb7p0^\xe6b,\x80\x1c$\xb5\xa7\xa4\xab']\xae&\xdf/\xa4\\\x8e._\x01̧\xa1'\xa4\x9f'\\\x0e,\xad4!\xcd\xfc\xb4\xf4\xf2\x04\x1c\xbe\x91\xf3\xf1F\xee\xc7[8 o\xeb\x82,:!\x8b\x9c3\xfb\xfa\xc5\xd1e\xa5sԳ\xc1\xf8TV\x9be\xb2\x81\xbfПs\xf0Em]\xe1\x85z\xf5L\xd3XH\xb9\xf9\xfa3\x83_\xb8\xcc==\x88\xa9:z\xbcwC\xd0\x1a\x16\xf1\x04\x81\xd6j\v\x15\xb0\xfc\xb5\x82\xc1\x92iW6ms\xf4W\x93f\r7,\xdb\x0f\xa0\xef\xa3~\xc2V\xe9\x82Y8o\xeed.=p\xfa\xfb|\r\xf0I5\x97^݊\n\x86\x17\xa58\x92\x1f\x10\x81y\xde\x05\xf12\x86\x882\x93\t\xe5\x9aB\xfd\x9a\x05\xdf\xef\xbe\xdf;r\x99W\x97\xee\xa9ᚸ\xe3\xc3\xe4\x11\xee\x1e\x9du\xe2\n\x86dm\xf1\x94`\u007f\xd4\xde߰\xb6\xcaϯ\u007f\xadg\xac\xd2l\x87\xbf*_\x81k\t\a\xfd\u07bd\xf2kAj\xd4\xd7\xec\xf5W\x181m\x1aj\x81\r\x80\xb5\xd93\xa3:P\xb4ʘ8\x999\x89֊\x85\xcd<<\xfc\xea7`y\x81돕\xbfB]\x95L\x1b$l\xd6\x1b\xf3\x836\xf4߽z\x8eE9T\xd8\xf3\xcf\xc3ukt\x19:\xee\xa6\xf6\xa4\xd5\x1fz\xf5\xc4j\x14-\xb1\xe8c|T\xc7E\xeb\x10ɟ\xf6(\x87N\xc1\xe9\x94Tt\xc1\v\xf7\x85\xcd\xeb\x16\xfc\x99\x92\xdfSE\xe7\\\xa1\xb5\xe5\xb2s\xbe\x1e[(2\x19\xf2\xbc*\xed\xaa\xf5\x84Zm\xae\xba\xcd\xcb*\xcf\xf9\xb4\x94^\xe1\xcfy:]\x8fG\xb8\xf2\x8e:\xefT\x9ek\n\x80=3Ӥ\xbeDUj\vΏt6-A\xc3\x1c\xf0\x80\x12\x94t\x99.\xae\x0e\x8e/A:\x1c\x13\x81څ\x12Ri\xaaR(\x96\xd7'\xbc\xd6^\xa1l僓_\xfa\x80\xfa\x9d\x99\x81ٔt\x8b a\xacP\xbc:\xb9\x82\x9cY\\E\x81&ɾ(\xb3e\x86\xf7\x19\xdd|\xb0\x96<\x84\x98\xd5<,\x1d85\xb2\xd6\xc4VY&@V\xc5ƫvVw\x88\xd1oT@Єܧ\x99\xe3\xe57ƥ\xc5\xdd(\xba8\xde\xd9u\xcd?'\xef\xac\x199\xb53Se\x19\x1a\xb3\xad\x84\x88\x19\xf9\r\xe7\xbe\xfe6]V\xdfb\xb53\xd7ɋ@\x97\x12X\x17\xd4\xf39\x81\x05\x1a\xc3vu\x99\xb3g\xd2@;\x94\xe8\f\xa0X\xe4ѻ\x88m\x0eY\xbfȗ\x8fe\xb1\xccV,LP\xe7\x00tz\xbd\x8b\xd9MB\xed|)D^\x17\x82\xadU\xf3\x898\xf9Vr\x9d\xa2\xcao\x9a\x8e\x84\x1b\x17\x86v\x84h\v\xf7\xa2\xe0;Nz\x90\x88\xb4cz\xc3v\xb8ʔ\x10\xe8\x12\xce\xc7\xebz\xcb\xc3\x1a2\xf5\xbe\"3\x8b[\xfb\xd4\xed\x1bb\x1e\x9eھF\x06\xf3\x85\x16]\x1dW\xcb5\xb6\x85\x91G\vRn\xe2\x93T\xb7\xc7B\xb4\x84\xf0x\xa5ݾ\xf5\x01\vr5Xҡ\xa2\xf0E0\x06\xe3~m\xc1~S\xfa\x02\n.\xe9\x1f\xb2\xfb]P\xa2\x1e|\xd2\xfa]\xf5\xba\x85u\xdfQ\x9f&a\xba\xa3H\xb1>\x10S\xa6jM\x92e\x19Y\xf5xi,\x13\x11A\xfa]iWξ%6\xc7\xfc/\x11\x83o\x84\xf0\xdbn\xff\xe6[\xf5F\x8d:p\x1es.\x8d\xdd+\x91\xa8J\x05\x97܌\x12\x9e5\xb7\x96\x04w\xf7\x96\x10,\x89j!\xc0\x90\xf0\x9a\xa8L8\xa7B\xdc{R\xf2\xb7\xd3Q˾#\xd5t\x9e\xb2\x11\xc2\xe6\x14\x91e\xe3P0\xb1-\xff\x99\x147\xf5X\"e\xb6grGL\xa5U\xb5\xdb\xd7|9\xa1\x82\xa7\x82~\x15-\nJw\xb0M}Cc+-;Q\x9fpg\x93w\x96˲\xa7ɕ\x86(t]\xb3\xff2\xd4\x1a\\m\xb5*V\x81\x16\xeeb\xe5\"Db4W\xe4h\xd8}\x14\xe5\xe0\x8b!\x87\xa2^\x8e\r\xca\x12%0\x13֓\xf0\r\xd7H\bvU\xb1\x9b.\xc1\xe0\x8c\xe0\xe3O|\xeb\xaf\xf12Z\xf5\x9f\xffቾ\x87$\xe3\xe8ݬ]\xe4L\x9e\xc6\xc0Y\xa8\x81}'\x90\f\x16\x83\xd87\xb9ޝd[\x1f^\xe6-\xbe\xa6\xabX\xff\x92\xc5\xeb8P\x87\x979\x89o\xe6!\xbe\xee\ue799\xab\x9e\xbft\xc6\xfe\x1a\xbaE\\\xc4\x00!\xe2$F\xb6Ѹ\x8d\x8bNb\xc7G\xac\xd78Q`{\xe07\xbe\x92\x97\x18\xd5\x03\xa3\x87N\x80杳\x1df\nO\xda\xc8\x1b\xcb2$v\xfd<\xfcŚs_\xfe\xbd\xfeQ\x1a\xf7g\xa6\xa4W\xb7\xe6\n\xfe\xe3\xbf\xce \x84v\x1f\xeb_\x9f\xa1\x87\xff\x17\x00\x00\xff\xffp,\xdd\xe3\xddg\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c9n\xef\xfa\x15(\xe5\xc1wU\x9aѺ\xf2\x90\x94\u07bc\xb2]Q\xed\xc6VY:\xdfC\x92\aN7f\x86+6\xd9G\xb2G\x9eK忧\b\xb2\xbf\xd9ݜ\xb1t\xb7we\xbe캇\x04A\x00\x04\x01\x10\x84.V\xab\xd5\x05+\xf9WԆ+y\x03\xac\xe4\xf8͢t\xff2\xeb\xa7\u007f7k\xae\xae\x0fo/\x9e\xb8\xcco\xe0\xb62V\x15_ШJg\xf8\x1e\xb7\\r˕\xbc(в\x9cYvs\x01\xc0\xa4T\x96\xb9\xcf\xc6\xfd\x13 S\xd2j%\x04\xea\xd5\x0e\xe5\xfa\xa9\xda\xe0\xa6\xe2\"GM\xc0\xeb\xa9\x0f?\xad\xffm\xfd\xd3\x05@\xa6\x91\x86?\xf2\x02\x8deEy\x03\xb2\x12\xe2\x02@\xb2\x02o`ò\xa7\xaa4\xeb\x03\n\xd4j\xcdՅ)1ss\xed\xb4\xaa\xca\x1bh\u007f\xf0C\x02\x1e~\r?\xd3h\xfa \xb8\xb1\xbft>\xfeʍ\xa5\x1fJQi&\x9a\x99\xe8\x9b\xe1rW\t\xa6\xeb\xaf\x17\x00&S%\xde\xc0'7E\xc92\xcc/\x00\xc2rh\xcaU@\xf8\xf0\xd6C\xc8\xf6X0\x8f\v\x80*Q\xbe\xbb\xbf\xfb\xfa\xaf\x0f\xbd\xcf\x009\x9aL\xf3\xd2\x12Qѽ#\x90-\v\x1c\xc1\xb8ܢ\xf6L\xdcjU\x10L\x94\xb9\x97>\x12]\xc1Q\x0e\xc9o\xaaM\xc1\xad\xe3\xfb_*4N\xc8\xd5\x1anI\xc5\xc0\x06\xa1*s'\x99k\xb8\x93p\xcb\n\x14\xb7\xcc\xe0\xab3\xc0Qڬ\x1ca\xd3X\xd0Վ\xc3Ξj\x9d\x1fj]6\xc1/\xaf\x10\x1eJ\xccz\x1bƍ\xe2[\x9eѶ\x80\xadҭ\xbe\xf0\xeaj\xdd\x03\x19߲\xaee\x86?HV\x9a\xbd\xb2N\xff\xaa\xca\x0e{\f\x10\xba}\xb8\x1b\f\xa8\x91\t\xa8\x91Z\xa9\f\xe6n\x9f=3n\x1dz#\x98\xe0\x00\xc1W\xd205<\xd24\x95\x01[iI\xbb\xf4\v\xb2\xfc\xf8\xa8\xfed\x10\U0008a135>+\xae`\x83[\xa51\x02W\xa3\x1b\xef:\xa3֎0\x86PR\x95]\xc3\xe3\x1e\x1d\x19Y%l\x90{n\xe0\xedOPpYY\\\x8f\xa0M0\xd8\x13\x85\xc0\xf8\x15\x98G\xf5\xd1xV-\x90\xef\xfdİ\x0e\x11\x9f\xf7h\xf7\xa8\xa1T\xb5\n\x8e\xacr\xcb\x05\x829\x1a\x8bE\xe0x\xad\xf86\x81\xfa$\x14B\x04\x10\x066\xc7\x1a\xe7\xf1:\xddy\xcb6\x02o\xc0\xeaj<\x9d'\xc3F)\x81l\xa8\x84\x87t\xf8\x82\xc6\xf2l\x81\n\x97C2\xf8Q\x11\"\xe8\xf0\x03\xad-B\x87M+k\x96=!\xb0\x9a\x1a\xeep\x10\xa2C\xc4\x1e\x05\xe0\xbf%\xbcw\x9a+s\xfad\x8c-\x04\xcd\xc5Q\x90\xb6\x94\n\x84\x92;\xd4~6w*)K_^\x85\x9c\x1e\xf13\x88\xe9\a\xd2\xf6\x92^m;:tc\xc8\t\xc2\xed\u06dd\xf7\xf0\x1a\xf6p\x03w\xd2\xf9-\x81\x1et#\u09db?\x1f\xfa\xad\xa8\f\x05\x89\xa5\x92+:*ױ\x99<\xb1\x13A*\xdd\xe3\xc8\x18\xb5fR?a\"\xd8Gw\x92\xf8\xf1\xfe\x8eC\xb0\f\xf3:\xc6I\x91yfq\xc73(P\xef\xe6\x0e\x8en+\x9d~OC!Q\xeb\xfav\xa2\x84\xa5\x1d\xedu\v\xaa;\x1a\x82뷕۹\t\xbdjf/v\x9d\b\xc8Ow]^\x11\x1d\xb1d\u007f,R\x97\xe59ݥ2q\u007f\x82\xc6?\x81\x17\xe3\xb3\xdf#\xe6OȂQ\x88\xf4\u007f\xdd1G\x02\xfd\u007fP2\xae\x13\xf6\xf0;\xba\x1a\x15\xd8\x1b\x1b\xa2X\xddi\xdc\f܀\xe3\uf049\xf1UOdq\xca\xe9\x16\x14\xfe Wۑ\xc5r\x05\xcf{e\xfc\x99J\xa1\xd9E\x90\xdc\xc0\xe5\x13\x1e/\xafFz\xe0\xf2N^\xfa\x03\xfedu\xd3X\vJ\x8a#\\\xd2\xd8\xcb\xef1\x82\x12%1\xa9\x1b]A\xa7\x9a\xcaΗ\xac-\x017\xb0\xb9wuf\xee\x1c\xd6IrX*\x13\xb9M\x9a@\xe5^\x19\xeb#\x8b=\xb3\xf4\x94(\x16x\x19\n\xd1+`[\u007f\xf3\xadt}\xa7\xe9\xd4\xde \xe0\xea\xb8f\xe65\xaccc\x13\x11\xf3@\x9dcu\xd9\xee`\xafO/\xfdE'M\xc222.\x16\xe1\x96Zeh̼\x88$h\xeb\x85 a\x13 dށ\xf1\x17\x86\xf3Aɺ\xa5\x1b\xa4\x8eH'\x9a\xf2\x1f\xbeu\xa2\x97n\xf3\xbb\u007f/\tߩx\x01\xed٢`Û\xf1$\x14o\xfd\xc8z\x9b\x04@\xde5л\x8a\xb6z\xba\x05\x19\x04\xe9\xf7pL\x17\\\xde\xd1\x04\xf0\xf6ŏ\xf5FI\xe29\x86\xfbm=\xb6%z\xf3\x81vo\xaaE\xa4(r\xaf\xb1ǹq\x9c\xdb\x19\x8a\x89 \xa5\xb2\xddp\x82\x83[\xaa\xfc\x8d\x81-\xd7\xc6v\x11M\x15\x8aja\xf7\xb7\xedT\xcfI~\xd0\xfa,\xc7\xe9\xb3\x1f\xd9\td\xed\xd5s\x9d_0y\x15\x1bkt)\x84\xc0\xb7\xc0-\xa0\xccT%)\xfc\xe2\xb6:M\xe1Y\xe0\x15t2\xc9\xd2\x14\x84k(\xab\"\x8d\x00+\x92:.g\xe34\xdd\xee\x1f\x19\x17\xaf\xc16;\x95\x86\x11k=\xb6\xd5\xf9\x18\xddD\x91\x82}\xe3EU\x00+\x1c\xe9Sݞ\xad\xcf\xe2\xe8q\xbc\xc9\xe5 \xb8t\x8cX\xe56U)Ц\xeeH\x9f\xb5ᶉ\xe196\as\x90\x02%\x81\xc1\x96q1qy>n'\xd1\xf6\x14_#(\x8b\x97s\"\xd2&_\x11)\x12\x02\xb1\x89\xc6⼶.u\xba\xa9x\xaf1\xcd<[\nJ\xd7\xe6Y\xa9\xb9\x93%\xf5\xd2\x16Z\x101&\x8f?L\xb4Q\xfba\xa2-\xb4\x1f&\xdad\xfba\xa2-\xb7\x1f&Zh?L\xb4\xba\xfd0\xd1~\x98hs\xdd\xe6\xb4\xf5\x12F\xfe\xc5\xc9ď\x8bX$\\Oϡ8\x03?dS\xdc\xfa\xd7'\xa9\x19\x96w\xf1Q\x91\xac\xe0\xf0\xaceE/rb\x12\xd0&]\xb4GI\x93r\xe96H-\xde>\x81~!\t\xf3;\xb2oS\x12~\x96\xd2|\xfay\xa6M\x9aM\x9dh\xaa\xeaI\"t\xa8_\xf68\xb3\xb7\x9bC\xd2\xcf\xd7!;\xb7\xc6\xf4\uf783\x9a\x90\x8a\xb3\x90\x803\x9f\x98;G\xaf\x81\xeb\xd1'\x98\xee%\x8c\xfe~\xe8e\xb1\xf8\\\x06QN{\xc4q\x17\x19\xb2\xf4\x8c#\xb2\x1e\xf2Q\xccQf{\xad\xa4\xaaLpk\x1c\xf4w\xe4]\x85\x1b\x17\xca6\x89\xbd\xbd\x88\xc0|\v{UE\x12Cgh\xb7\x90&4\x9d\x1c\x14\xae\xc2в\xc3\xdbu\xff\x17\xabB\xaa\x10\xecɊi\xde\xffJK&:;\x85\xa8\x9f\"4qF\x9dzg\x96\x9e)\x9d\x9e$4\x9f\xd5sJj\xd00\xf1g\x12\xe8rBP\x8a뼐\xfcsF\xcaOb\xba\xe7w\xdf\f\xa6$\xf5\x9c\x95ʳ\x98\x11\x99\x98\xc0\xd3O͙\ayB\xdaN\x12q\x96StNN\xcc\t\x890\xb3\xebHNlj$\xda\xcc\x02\x9eL\u0099K\xafY\bˍSoғjfAS\xc2\xcdr*\xcd\xcb%̾\x84\x130\xadj\x16\xd3a\x16\x9d\x84y\xfc\x16\x13^NIsY\xa4ؙ)-M\xca\xcaļ\xa7&\xb2\xf4\x13U&\x80\xa6\xa4\xafL\xa4\xa7L@\x9cMZIMJ\x99\x80\xbdp\xec\xceJ\xc9̏\xf1\x97аx\xbe\x89\xbf\x95D\x9d\xbb0\xa5{\xe6⒋\xf2y\xd0\xdd\U00072d9a\xe6\xcdϘ\xe5\xc9\xed\xfet\U000f3a04奠\xfb\x8c\x03ϣ^\xb3\xdd\xe3\xb1y\xd7\xfa\x9b\xa2w^\x9b#A\xfa\xfc\xa5\x11\xcf\xf5\xc0\x88f\x06\x9eQ\b`1\xe1\x1a\xad<\xf3\x8f\xf93\xb5B\xa7\xf3݆\v/vÛ\xff+/\xc1\xf4\x94-\x16\xf2\xb5{,\x1c\x94\xfa\xe9\xef\t\xfe\u05fc\x81\xe8mY\xfa\xf6\x97\n\xf5\x11\xd4\x01uk1,<\xa4\xf0\x1b\xcd8\xef\xa6\xde\xfaA\u007f\xf8\x12\x12\x03ù\xddp\xf0N\xfa#,\nv\x80#\xc1q{^4\xbcv\xea\xcd\xf9\x01\x13]\xe3\x91\x1fՌ\x8e\xfc\xbed{\xa6\xbeBx]\xd7\xe1t\xe7a\xf1\xd8~\x15\a\xe2|\x17b\x06dꫂ\xb4\x1b\xb8\xc5W\x04\xaf\xe5J,9\x13\xc9VT\xda+\x81\xd7x\x1dp«\x80\x13\x9c\x8a\xd3܊d2\xa5d\xff\xbf\x8as\xf1\x8a\xee\xc5k8\x18\xe7\xb9\x18\v \aY\xfd)\xf9\xfaI\xb7\xcb\xc9\x17,)\xb7\xc3\xcbw \xf3y\xf8\t\xf9\xf7\t\xb7#K\x98&\xe4ٟ\x96_\x9f@\xc3Wr>^\xc9\xfdx\r\a\xe4u]\x90E'dQrf\u007f>;\xbc\xaet\x8ez\xf66\"U\xd4f\x85l\xe0/\xf4\xe7\x1c\xc4\xe6\xeb\x127\xaeW\xcf4\x8d\x85\x94\x9b\xe7\xaf\x19\xfc\xc2e\xee\xf9ᄪs\x8e\xf7\xaeHZ\xc3\"\x1e\xa1o\xad\xb6P\x02\xcc߫\x18,\x99\xa6\xbaq\x9b\xa3\xbf\x9b5k\xf8\xc0\xb2\xfd\x00\xfa>\xea'l\x95.\x98\x85\xcb\xe6R\xea\xda\x03w\xff\xbe\\\x03|Tͭ_\xb7\xa4\x84\xe1E)\x8e\xce\x0f\x88\xc0\xbc\xec\x828O \xa2\xc2dB\xbd\xaaP\xc0g\xc1\xf7{\xe8\xf7\x8e\xdcfֵ\x8bj\xb8&\xee\xf80y\x84\xfb\xafd\x9dPŔ\xac\xad\x1e\x13\xec\x8f\xda\xfb\x1b\x16\x97\xf9\xf9\xe5\xef5\x8dU\x9a\xed\xf0W\xe5K\x90-Ѡ\u07fbW\u007f.h\x8d:Ϡ~\x86\x12;MC1\xb4\x01\xb06}hT\b\xcba\x19S'3;\xd1Z\xb1\xb0\x98\xc7\xc7_\xfd\x02,/p\xfd\xbe\xf2\x17\\\xab\x92i\x83\x8e\x9a\xf5\xc2\xfc\xa0\x8d\xfb߽z\x8eE9TX\xf3\xcfC\xbc5R\x8a\x12]U\x9f\x84\xfd\xa1WP\xad&ђ\x88~\x8d\x8f\xea\xb8h\x1d&\xf9\xdd\x1e\x95\xd0)8\x9d\x9a\x92\x14\xbc\xa0'F/[\xf1hJ\u007fOUݣJs\xcbu\xf7|A\xbaPe3$\xbaU\x9a\xca\x15\x85buT\xde\xe7\xac\xd2{ts\xdaޘ6\xf7\xb1杵\xce\xe0\x8c\x19a}\x0fu\x11@\xad߭\xb2L\x80\xac\x8aM4\xa2\xe4\xd6\xd0\f!\xb4for}\x82\xd1\f\v=ѹ\xb4\xb8\x1b\xcd7\xb9\xeaې\xa6t\xf6\xaa\x1b\x00\xe9\xab6U\x96\xa11\xdbJ\x88c\x93'\x95H\x82hhꥉ\xf2\x91qq>E\xfc\xe8\x189&N\x98SY\x1f*c\xa0\xcc\xc3֎\x1e[>\x8f\xf04\x8a\x04f\xf4\xca\x02\xcfS\xe1v<\x82\x8a\xbf꼓\xd0Д\a|f\xa6ex\xcc\xdel\xc1\xf9\x91\xe4\xf09h\x98\x03\x1eP\x82\x92\x94\aGU\xb2|\x81\xe2\xe1\x98\b\xd4.\x94\x90hW\x95B\xb1\xbc>\xfej\xd3.\x14\xb5}\xa4\xc3]\x1fP\xbf130\x9b\x82\x8f\x11\"\x8c\xad-ok\xdd@\xce,\xae\xa2@\x93\f\x83\xa8&\xce\f\xef\x9f\x02\xc9\xda\xec\xf6\xe1nj\xe4\x84\x04\xb7J+ƿQy\xd1\xefT\\㕥j\xac\xf1\xcafU\xd5P1E\x16ת\xaa\x17_&\xed\xd5\xc5Z\x88\xd4\xc9\xdb\a\x940\\\x97\xdb\xf4\x19\xc3\x05\x1a\xc3vu\x11\xc4gg\x9e\xedP\"y\a\xb1\xb0\xbc\x8f\x9f\xb4\x19\xa6\xfd\x12\x80>\xd0\xcb2[\xb10A\x9d \xd3\xe9\xf5&\xe6T\b\xb5\xf3\x85Ry]&\xba\xb6[O\xa4ɷ\x92\xeb\x14;\xf7C\xd3\xd1ц\xeeh\x88\x11mYo\x14|ǝ\x91蘴cz\xc3v\xb8ʔ\x10H\xbav\x8c\xd7kn\u0590\xc7\xfb\x05\x99Y\\\xda\xc7n\xdf\x10\x10\xf4\xdc\xf6\x15t\x98\xcf\x15\xa3*ϖkl˦\x8f\x10R4\xf1Iv\xad\xa7B\xb4\xc0\xf8\x18\xd3n\xdfz\x83\x05\xbd\x1a\xdc\xccPo\xfc*xJ\xf1\xa0O\xc1~S\xfa\n\n.\xdd\u007f\x9cSL\x11\xbbz\xf0I\xf8Sm\xcb\x05\xbc\xef]\x9f\xe69E\xc7\xca\xc4zCL\xf9q\xf1\x14\xfa\x15|±\xdb\xe1\xb3\xe21\xa7\x18u\xac\xaa\xba\xebr'\xef\xb5\xdai4\xe3]\xb5\x82?3n\xb9\xdc}T\xfa^T;.[{\xe3\xa4\xce\xf7L[΄8z|b\x88r\xc9\x04\xff+\x97\xbbw[\x8b)\xb3-\x8cX\x9e\xb2Q̑\xdf\x12\x10\x9e\xfa\xe1=\xbaCy\xd2m\x89\x8bL\xe0\xc0\x92Ԅnm\xf4\x8dK/唫\xbfQ\x95\xed\xa9\xc9V\xcdFD\xbe\x9es\r\x9f\x94\xc5\xfaR\x87\xf7a\xba\x83\x05\x8d]\xe1v\xab\xb4\xf5\xc1\xbe\xd5\n\xf86\xb8A\xb1\xe8\x0e\xe3\x82.\xa5}\x01w\xe0\xb6\xcd\xcbiw&E84)\x18\xaarW\xb0\xa3\xcf\xdbfY\xe6\xbcl\xbc6\x96\x89\x88\xee\xfe\xae4H\xf27\xdd\xce\xc2\xfcO\x11\x1bsD\xf0\xbbn\xff\xa6xFsr\x138O9zW\xe3ϭ\xe8)\x0e\xf4\xda\x02%Jr\xf0\xd5\xd9C\x95A\x12\x83\xb2D\t\xcc\x04|\x12\x1e\x95γu.4i\x99\xb6\xa9\x1e\xd3C\xaf\xf3\x82\xb3D\x90\xe3\xf8>\x84ȯ\u007f\\{;\xfcs.W`\xb8\xac\xff~\x89\x8f+{Q0·\xd2HA\xba\xe8\x1d\xf6\xc8\xfb\xe9\xf9:}\xf4\xff\xb6nΡ9=?\xa4\xd8\xcb_\a\xdd\a/\x14\xa8L\u007f\xd3%ظ\x11z\xfc\x81o\xfd\xb5z\xe6\xb0\xfe\xe3\xdf\xfd\xe5\xc1!\xc9\x1e{3k\x8a\x91\x95\xd5\xd8T\vE\xf9\xef\x05:\x1b\xc9 \xf6\xad\xbc7'\x99\xf3\x87\xf3\x1cԗ\xf4N\xeb?\xad\xf32>\xdb\xe1<\xbf\xf4՜җ]\xdd3\xa3?籴\xc7\xfe\x1c\xbaE\xbc\xd2\x00!\xe2\x97F\x96\xd1x\xaa\x8b~i\xc7-\xadq\x9c\xa8\xf8?pU_\xc81\x8d\x9e\x03\xa3\x8f\xa4@\xf3\xce\xde\x0e3\x85/m$\x9ce\x19:q\xfd4\xfc\x13Z\x97\xfe\xefQ\xd4\u007f%\x8b\xfe\x99)\xe9\x8f[s\x03\xff\xf5?\x17\x10\xaeZ\xbe\xd6\u007f\x0e\xcb}\xfc\xff\x00\x00\x00\xff\xff\xed\xa36\x94nl\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKo#\xb9\x11\xbe\xebW\x14f\x0f\xbe\x8cZ\xb3\xc9!\x81.\x81F\x93\x00\x83x\xd6\xc6\xc8q\x0eI\x80\xa5Ȓ\xc45\x9b\xec\xf0!\xad\x12\xe4\xbf\aŇ\xba\xd5ݲ\xe4A\xb2ˋ->\x8aU_\xbdٓ\xe9t:a\x8d|F\xeb\xa4\xd1s`\x8dğ=j\xfa媗\u07fbJ\x9a\xd9\xfe\xfbɋ\xd4b\x0e\xcb༩\xbf\xa23\xc1r\xfc\x84\x1b\xa9\xa5\x97FOj\xf4L0\xcf\xe6\x13\x00\xa6\xb5\xf1\x8c\xa6\x1d\xfd\x04\xe0F{k\x94B;ݢ\xae^\xc2\x1a\xd7A*\x816\x12/W\xef?T\xbf\xab>L\x00\xb8\xc5x\xfcI\xd6\xe8<\xab\x9b9\xe8\xa0\xd4\x04@\xb3\x1a\xe7\xb0f\xfc%4\xce\x1b˶\xa8\fOwU{ThM%\xcd\xc45\xc8\xe9\xea\xad5\xa1\x99C\xbb\x90(d\xb6\x92H\x1f#\xb1U\"v\x9f\x89\xc5u%\x9d\xff\xf3\xe5=\xf7\xd2\xf9\xb8\xafQ\xc12u\x89\xad\xb8\xc5\xed\x8c\xf5?\xb4WOa\xedTZ\x91z\x1b\x14\xb3\x17\x8eO\x00\x1c7\r\xce!\x9en\x18G1\x01ȘEjS`BD-0\xf5h\xa5\xf6h\x97F\x85Z\x9f\xee\x12踕\x8d\x8f('Y \v\x03E\x1ap\x9e\xf9\xe0\xc0\x05\xbe\x03\xe6`\xb1gR\xb1\xb5\xc2\xd9_4+\xffGz\x00?9\xa3\x1f\x99\xdf͡J\xa7\xaaf\xc7\\YM:z\xec\xcc\xf8#\t༕z;\xc6\xd2=s\xfe\x99))NZ\a\xe9\xc0\xef\x10\x14s\x1e\x8c\xb6e,\x1e?\x97ț\x1c(\xfb[ƪ\x82E\xf6\\\xb3\x81\x0f \xa4\xa3\x02\xc0E\xa2C\xb0\xa8<\xa3\xf59x\x1b\xde$>7z#\xb7C\xa1\xbb5\xcd%\x8b\xb9B\xba\x87\xdc2\xdeD\xa1\x89\xac\xa3\xb1f/\x05\xda)\xf9\x87\xdcH\x9e9\t6e\xae\x8dD%\xdcP\xd2\v^\x16E\xb1(ȫ\x99\xba\xa2\xc3\xe5ic,\x8d\x99\xd4ɂ[\x021\xd8\xd8:\xa7T\xedQ\x8bS5rƍ\x89Qˡ\x80\x83\xf4\xbb\x14\x0e\u0558\xdf\xc1\xab\xbeG\xe3\x05\x8fc\xd3=ޟvH;S\x02Ep\xc8-\xfahm\xa8\xc8|Ȕ*\x80/\xc1ŀڏ\x13e\xc4B\xad\x9c~\xc1\xe3\x10h\xb8\xa6\xdc\\\xc2\\g\xf9\x8eJ\xe7°\xc5\rZ\xd4~4\xa8Sgb5z\x8cq]\x18\xee(\xa4sl\xbc\x9b\x99=ڽ\xc4\xc3\xec`\xec\x8b\xd4\xdb)\x01>\xcd\x1e4\x8bm\xc5\xec\xbb\xf8\xe7\x82\xc8O\x0f\x9f\x1e\xe6\xb0\x10\x02\x8cߡ%\xadm\x82*\x86֩o\xde\xc7\x1c\xfb\x1e\x82\x14\u007f\xb8\xfb\x16\\L\x93<\xe7\x06lV\xd1\xfa\x8fT\xa8E\xa6\b\xa2UҊ\xb1@\x99\x92\x94]gm\xa6X3f\x88c\x15fwP`\xa2\f2\x16Q_p\x18L_q\xb3\\\xec^\xf1\xb1RHK-$\xa7B\xec\xdc7J\x83!\xce\xea\xed\x11\xc1\xfa\x15\xf8\xa5\x880.x\x12 \xe7\xc3+\x1c?t\xf7\xb6mY\nO9\xc79\xf4T@9\xd0H9\x90\xd9!r1(p\xa35y\xa37\xc0N\xa1\xee\xce\xf5c\xfc\x1b#\xc4:\xf0\x17\x1c\x01~ \xcaǸ\xb1`\x9c\x8e\x11/\xc1a\f\xbe\xd7\u0600\xeb6\xce\xd9\x12\xed-\xbc,\x17\xb4\xf1\x94&\x19,\x17\xb0\x0eZ(,\x1c\x1dv\xa8\xa9C\x90\x9b\xe3\xf8]4\x9e\xeeW\x05\xd5Xa\xe4\x1a\xbf`;.C\x8a\xe1sX\x1fGj\x82\x1b\x84l,n\xe4\xcf7\b\xf9\x187\x16\xc0\x1b\xe6w \xb5\x93\x02\x81\x8d\xc0\x9f\x8a\xb5\v\x82\x9e\xf2\xffC\x8e\"ߠ\x9e\u05fc=\xb1\xf3\x16\x87/\x18_\xf1\x9fǼ\xed\x84B\xf9\x9d#\xffy-xɏG%ڟ\x1e\f\xfe\x94*,>\x92*Ϙy\x1e\x9ex\xa5R+\xcf\x16c\xceLu\x81\xb1\x16]c\xb4\xa0\xe6\xe9\xb6:\xade\xf9\u007fW\xad\x8d\xabuz\x1e\xe5zkE\v7\xb5*\xf1\x89\xe6\xcd\xcdJz\xb8\xea\xb6\x02f\xed\xa8Sl\xfb\x95\x9e\x8c\xbfH\x9b\xf2\xaeӧP?\xac!\xe8X\xa9Ō_\xc1\xdf5|\xa2ޖ\xb2\x93\x98\x13\xdfv\xcc\x00\xa4\x03m\x0et\xbcC/\x92\x00\xa3S\xbe\xa6n\x8di\x91\x9b\xe1\xb8t\x90JQƶX\x9b\xfdhƦBӢ:\x02sd:\xfb\xdfT\x1f\xaaw\xbfZ\x17\xa4\x98\xf3\xd4Ԡ\xf8\x8a{9|\xe5\x19\xa2{?8Q\x1c\xff\xe4\x0e\xf4\xe3\xc7\xd2,\xcfl\xde\xf6\xe3\b\x18\x1b\xa9\xa8\x16\x1c\x89\x13m\xc50|\x8f\xfc\xb8\xba\xbfs\xb1\x84G\xed\xc7ʾ\x03Z\x8c\x1d\x13\n\xaa\xe2M~\x97\bΣ\x1d1\x80\x93\xf6\xa2\xceA\x19\xbd\xed9N\x1a\xf9\x95\x82*\xb4dPƂ@O\xa9Io\x81\xef\x98\xdeb\xfb\n\x95\xf9\u007f\x9dS2\x9f\x9eʹ\x16\"\xf5%\xf3\xb8I\xa3Or\xacL\x1f\xbc\x00\xb7\x9b\xc7_\u007f\v\xf7E\xb3\x17ۜ+\xb8\x0f\xf6\x97,M\xa0N}\xfb\"\u070eooo\x87\xcf\xcd7 \xf1ַ\xf0W\xde5\xe0\xc0\\\xfb*\xfe\xeb\xe1PS\xb5z\xb5\x04\xfe\x92v\xa5\xe7\xc3|\x04\xd8\xda\x04\xff\x9agލ\x19t~\xee\u007f\v\x8f\xf1#Ƶ\"\x83\xf6\x14\x8d\xf0`\xa9\x95l_\xc5bP\x18\xcb-\xb7?/-z\xdfZ\xbak\xc3/17\xc85\x9ak\a\x93)_v\xf4\x9aA\xee΄\xf5\xe9\xa5x\x0e\xff\xfeϤMה\x13\x1b\x8f\xe2\x87\xfeǵw)d\x94/d\xf1'\xa7:&}\x1d\x84\xbf\xfdc\x92\xaeB\xf1\\>i\xd1\xe4\u007f\x03\x00\x00\xff\xff\x1d\r\x93\v\x97\x1c\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96Ms\xe36\x0f\xc7\xef\xfa\x14\x98}\x0e{y$\xefN\x0f\xed\xe8\xd6\xcd\xee!\xd36\xe3I2\xb9tz\xa0I\xd8\xe2F\"Y\x00t\xeav\xfa\xdd;$%\xbf\xc8v6=\x947\x91 \xf0\xe7\x0f\x04Ī\xae\xebJ\x05\xfb\x84\xc4ֻ\x16T\xb0\xf8\x87\xa0K_\xdc<\xff\xc0\x8d\xf5\x8b\xed\xc7\xea\xd9:\xd3\xc2Md\xf1\xc3=\xb2\x8f\xa4\xf13\xae\xad\xb3b\xbd\xab\x06\x14e\x94\xa8\xb6\x02P\xceyQi\x9a\xd3'\x80\xf6N\xc8\xf7=R\xbdA\xd7<\xc7\x15\xae\xa2\xed\rRv>\x85\xde~h\xbeo>T\x00\x9a0o\u007f\xb4\x03\xb2\xa8!\xb4\xe0b\xdfW\x00N\r\u0602\xc1\x1e\x05WJ?\xc7@\xf8{D\x16n\xb6\xd8#\xf9\xc6\xfa\x8a\x03\xea\x14xC>\x86\x16\x0e\ve\xff(\xaa\x1c\xe8sv\xf5)\xbb\xba/\xae\xf2joY~\xbaf\xf1\xb3\x1d\xadB\x1fI\xf5\x97\x05e\x03\xb6n\x13{E\x17M*\x00\xd6>`\vwIVP\x1aM\x050\xf2\xc82kP\xc6dª_\x92u\x82t\xe3\xfb8Ldk0Țl\x90L\xf0\xb1\xc3|D\xf0k\x90\x0e\xa1\x84\x03\xf1\xb0\xc2Q\x81\xc9\xfb\x00\xbe\xb2wK%]\vM\xe2\xd5\x14\xd3$d4(\xa8?ͧe\x97\x04\xb3\x90u\x9bk\x12X\x94D\x9eD\xe4\xb8\xd6;\xa0#\xbe\xa7\x02\xb2}\x13:ŧ\xd1\x1f\xf2µ\xc8\xc5f\xfb\xb1\x90\xd6\x1d\x0e\xaa\x1dm}@\xf7\xe3\xf2\xf6黇\x93i8\xd5z!\xb5`\x19Ԥ4\x81+\xd4\xc0;\x04O0x\x9a\xa8r\xb3w\x1a\xc8\a$\xb1\xd3\xd5*㨪\x8efg\x12\xde'\x95\xc5\nL*'\xe4\fm\xbc\x04hƃ\x15\x98\x96\x810\x102\xbaR`'\x8e!\x19)\a~\xf5\x15\xb54\xf0\x80\x94\xdc\x00w>\xf6&U\xe1\x16I\x80P\xfb\x8d\xb3\u007f\xee}s:g\n\xda+9\xe4g\x1a\xf9\xd29\xd5\xc3V\xf5\x11\xff\x0f\xca\x19\x18\xd4\x0e\bS\x14\x88\xee\xc8_6\xe1\x06~I\x98\xac[\xfb\x16:\x91\xc0\xedb\xb1\xb12u\x13\xed\x87!:+\xbbEn\fv\x15\xc5\x13/\fn\xb1_\xb0\xddԊtg\x05\xb5D\u0085\n\xb6\xce\xd2]\xee(\xcd`\xfeGc\xff\xe1\xf7'Z\xcf.H\x19\xb9\xd0_\xc9@*\xf3\x92\xf6\xb2\xb5\x9c\xe2\x00:M%:\xf7_\x1e\x1ea\n\x9d\x931\xa7\x9f\xb9\x1f6\xf2!\x05\t\x98uk\xa4\x92\xc45\xf9!\xfbDg\x82\xb7N\xf2\x87\xee-\xba9~\x8e\xab\xc1\nOW2媁\x9b\xdcbSQ\xc7`\x94\xa0i\xe0\xd6\xc1\x8d\x1a\xb0\xbfQ\x8c\xffy\x02\x12i\xae\x13ط\xa5\xe0\xf8\xef07.Ԏ\x16\xa6\xf6}%_\x17\x8a\xf6!\xa0N\x19L\x10\xd3n\xbb\xb6:\x97\a\xac=\xc1Kgu7\x15\xed\x8c\xee\xbe\xc0\x9b\x93\x85\xcb\x05\x9dơM\xceW\xae\x1e\x1er\xee,\xe1\xec\x16\xd6p\xd6s_璛\xe1\xbf$S:\xf1\xc8FG\"trԟեMoe\x81D\x9e\xcefg\xa2\xbed\xa3\xfc\x04P\xd61(\xb7\x1b7\x82tJ\xe0\x05)\x95\x81\xf61\xf5\x194`\xe2\x19\xbf\x11\xcb\xf1\xbf$\x90\xd7\xc8ܜ\xd9Y\xc1ႦW\xb2\x93Fz^\xa8U\x8f-\bE\xbc\x92YE\xa4v\xb3\xb5\xfc\xcf\xfa\x06\x82e\xb2\xb9\x94\x83\xfd\u007f\xfa\x9bIȸ]\x1c\xce#\xd5p\x87/\x17foݒ\xfc\x86\x90\xe7W>-.\v\xbd\xfdc\xe0\r\x94.^ʳIN\xfd\xce\x1cQd\xf1\xa46\xc7\\9\xae\xf6\xfd\xbb\x85\xbf\xfe\xae\x0e\xf7Zi\x8dA\xd0\xdc\xcd_i\xefޝ<\xb7\xf2\xa7\xf6\xae\xbc\x8c\xb8\x85_\u007f\xabJ(4O\xd3\xeb)M\xfe\x13\x00\x00\xff\xff--\nM\xde\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WO\x93۶\x0f\xbd\xfbS`\xf2;\xe4יHN\xa6\x87v|k79\xec4M3\xebt/\x9d\x1eh\n\x96إH\x96\x00\xbd\xd9~\xfa\x0eH\xc9\u007fdٻ9\x947\x81 \xf8\xf0\xf8\x00R\x8b\xaa\xaa\x16*\x98{\x8cd\xbc[\x81\n\x06\xbf2:\xf9\xa2\xfa\xe1G\xaa\x8d_\xee\xde-\x1e\x8ckVp\x93\x88}\u007f\x87\xe4S\xd4\xf8\x1e\xb7\xc6\x196\xde-zd\xd5(V\xab\x05\x80rγ\x123\xc9'\x80\xf6\x8e\xa3\xb7\x16cբ\xab\x1f\xd2\x067\xc9\xd8\x06c\x0e>n\xbd{[\xffP\xbf]\x00\xe8\x88y\xf9\x17\xd3#\xb1\xea\xc3\n\\\xb2v\x01\xe0T\x8f+h\xfc\xa3\xb3^5\x11\xffNHL\xf5\x0e-F_\x1b\xbf\xa0\x80Z6m\xa3Oa\x05\x87\x89\xb2v\x00T\x92y?\x84\xb9+a\xf2\x8c5Ŀ\xcc\xcd~4\x83G\xb0)*{\x0e\"O\x92qm\xb2*\x9eM/\x00H\xfb\x80+\xf8$0\x82\xd2\xd8,\x00\x86\xdc3\xacj\xc8n\xf7\xae\x84\xd2\x1d\xf6\xaa\xe0\x05\xf0\x01\xddO\x9fo\xef\xbf_\x9f\x98\x01\x1a$\x1dM\xe0\xcc\xe0\x043\x18\x02\x05\x03\x02`\xbf\a\x05ʁ\x8al\xb6J3l\xa3\xefa\xa3\xf4C\n\xfb\xa8\x00~\xf3\x17j\x06b\x1fU\x8bo\x80\x92\xee@I\xbc\xe2\nַ\xb05\x16\xeb\xfd\xa2\x10}\xc0\xc8fd\xb9\x8c#q\x1dY'\xc0_Kn\xc5\v\x1aQ\x15\x12p\x87#?\xd8\ft\x80\xdf\x02w\x86 b\x88H\xe8\x8a\xceN\x02\x838)7dP\xc3\x1a\xa3\x84\x01\xea|\xb2\x8d\x88q\x87\x91!\xa2\xf6\xad3\xff\xecc\x930$\x9bZţ\x1c\x0e\xc38\xc6蔅\x9d\xb2\t߀r\r\xf4\xea\t\"f\x9e\x92;\x8a\x97]\xa8\x86_}D0n\xebW\xd01\aZ-\x97\xadᱨ\xb4\xef\xfb\xe4\f?-s}\x98Mb\x1fi\xd9\xe0\x0e\xed\x92L[\xa9\xa8;è9E\\\xaa`\xaa\f\xdd\xe5ª\xfb\xe6\u007fq(Cz}\x82\x95\x9fDf\xc4Ѹ\xf6h\"k\xfe\xca\t\x88\xea\x8b`\xcaҒŁh1\t;w\x1f\xd6_`\xdc:\x1fƔ\xfd\xa2\x9c\xfdB:\x1c\x81\x10f\xdc\x16c9Ĭ<\x89\x89\xae\t\xde8\xce\x1f\xda\x1atS\xfa)mz\xc34\x8aYΪ\x86\x9b\xdci`\x83\x90B\xa3\x18\x9b\x1an\x1dܨ\x1e\xed\x8d\"\xfc\xcf\x0f@\x98\xa6J\x88}\xd9\x11\x1c7ɩsa\xedhb\xecd\x17\xcekR\xea\xeb\x80ZNO\b\x94\x95fkt.\r\xd8\xfa\b\xeaP\xf9\x03\x81\xf5I\xe4\xf9\xca\xcd\xe0Tl\x91\xa7\xd6\t\x96/\xd9I\xb6\u007f\xec\xd4i\xa3\xf9?\xd6m-\xbd\x82\x06 \xa5{|W\x9fE\xbc\x8c\x01f\xd5;\x8bd\x14\xb1\xd0 \xbcJ+\x90&u\x8c\xe9|k\x19\xe8R?\xbfA\x05?g\xcc\x1f}{u\xfe\xc6;\x16\xb9_u\xba\xf76\xf5\xb8v*P\xe7\x9f\xf1\xbde\xec\u007f\v\x18\xcbUz\xd5u\xbc\x91\xf7\xb7Թ\xe3\x1dJ/\xc7\xcbY\f\x0ewH\xc9^D68\xbd\b\xda\xcd\xfa\xf6[\x92\xbe\xe0~\x95\xd6\v\x856\x8e|\xa1>\xaf\x1a\xb9\x92G\xd5Ȓr\xcb \xc8C%:d\xa4C\xc3{4\xdc\xcdF\x04x\xec\x8c\xee\xf2\xc2,9\xe9\xa5D^\x9bܙ\xbe\x1d\xbeT\xaa\x898#\xfb*\x97ÌY\xc0\x9f\x99/\xf4\x97K\x1bTCͿ\xa8G\xb1\xe2D\xdfХ\xb2\xffH\xb5N1\xa2\xe3!J\xbe\xb5\xa7\v^ڦ\xc6\xda\xfe\xfd\xee\xe33\xbd\xea\xfd\xc13\xbfK\x95q\x05M\x88X\x91i\xe5\xad!sҭr\x179'\xa3\x8cӷ\xcf)Q\xb3'\x8a_\x83)\x05\xf3\f\xc4\x0f{\xc7\xd2Rѕ\xebr\xfa\xba\xcb\x01\x91\xf2SD\xab\xe9#H\xc6\x06\xa1A\x8b\x8c\rl\x9e\xca\xdd\xf0D\x8c\xfd9\ueb4f\xbd\xe2\x15\xc85Z\xb1\x99\x91\x91\xbc\xc0\xd5\xc6\xe2\n8\xa6K*\x9bM\xf38\xfa\x01\xed\xe4\xfa\x8b\x99\xdb\x1e\xd1~[aޞ\xc3\xf4\xee\xfb\xe0@\xcb\x1a[\xb6\x8e+\x95F\xf9\xf6\xf6\xfa\xfe\xcfw\xa3a\x00m\x94F\xe3Dr\xe8\xe1\x1bı\xc1(\x8cY}A\x04\xc3*\xe0\x14\xc0\xd0\x06\xab\bc\xc8#\x86 \x0eaIu\rZ\x94nȒ\xf4\xa9-0\tj\xf3\v\x96\xae\x80;4D&\t\xa6Tr\x87Ɓ\xc1RUR\xfc\xb7\xa7mI\xd7\xe8І9\x8cq\xe5\xf0y\xd7/Y\x03;\xd6t\xf8\x1a\x98\xe4в=\x18\xa4S\xa0\x93\x03z~\x89-\xe0ge\x10\x84ܪ5\xd4\xcei\xbb^\xad*\xe1R\xfc.U\xdbvR\xb8\xfdʇb\xb1\xe9\x9c2v\xc5q\x87\xcdʊj\xc9LY\v\x87\xa5\xeb\f\xae\x98\x16K\x0f]\xfa\x18^\xb4\xfc\x1b\x13#\xbe\xbd\x18a\x9d)F\xf8|x=!\x01\n\xb0 ,\xb0\xb85\xdc\xe2\xc0\xe8\xe4 \xdf\xff\xe3\xee\x03\xa4\xa3\xbd0\xa6\xdc\xf7|?l\xb4\a\x11\x10Ä\xdcbt0[\xa3ZO\x13%\xd7JH\xe7\u007f\x94\x8d@9e\xbf\xed6\xadp$\xf7\xffth\x1dɪ\x80K\x9fԐ\xc3\xec4i./\xe0Z\xc2%k\xb1\xb9d\x16\xbf\xba\x00\x88\xd3vI\x8c}\x9a\b\x86\xf9\xd8tq\xe0\xda`\"%MG\xe45Ʉ\xee4\x96$=b \xed\x14[\x11=\x14\xb9s6]^\x8c\b\xe7\r\x97\xbe\xacw\x9a.\x82\\p\x99\xecI\xd8\xe4\xc0\xa7&\x87\x19VΈ\x024S/\xdb\xef\x19F.\x1b\x1dl1\xa3pD\f\xf4I\xc5\xf1\xcc=n\x14\xc7\x1cl\xda\n\xaefA[)\xe3#\u007f\xd4I9?\x85>%\x9f\x05L+~\x06W<\x91\x81\xc1-\x1a\x94%&\xc7u*\x9d\xc9 \x1b&\x1as\x8cǕ\x02Nx\xf5,ⷷ\xd7ɓ'&F\xecn~\xee\x19\xfeз\x15\xd8p\x1f\xe8Ο}q\xbd\r\x87y\x9f\xe6\x140\xd0\x02Cb\xda\a\t\x10\xd2:d\x1c\xd46K\x91\xca' \xc37\x18w\xbc\x0e\x1e,\xba\xcaCh!\xde\x03#\xdf)8\xfc\xeb\xee\xdd\xcd\xea\x9f9\xd6\xf7\xb7\x00V\x96h}^\xee\xb0E\xe9^\xf7\xa5\x02G+\frJ\xfc\xb1h\x99\x14[\xb4\xae\x88g\xa0\xb1\x9f\xde|\xces\x0f\xe0Ge\x00\xbf\xb0V7\xf8\x1aD\xe0x\uf593\xd2\b\x1b\xd8\xd1S\x84G\xe1j1\r\xa6=\aH\xbd\xe2\xb5\x1f\xfdu\x1d{@P\xf1\xba\x1dB#\x1ep\r\xaf|Zs\x80\xf9+\xd9\xceo\xaf\x8eP\xfdS0\xedW\xb4\xe8U\x00\xd7\xc7\xe1\xa1\xd1\x1d@\x06\xcb3\xa2\xaa\xf0\x90UM?\x1fT\xc8U\u007f\v\xca\x10\a\xa4\x1a\x90\xf0\x84Iz\xc1Q\"\x9f\x81\xfe\xf4\xe6\xf3Q\xc4c~\x81\x90\x1c\xbf\xc0\x1b\x10\xb1\xd8Ҋ\u007f[\xc0\a\xaf\x1d{\xe9\xd8\x17:\xa9\xac\x95\xc5c\x9cU\xb2ه\xf6\x881\tʄyp\xcdL\ueffa*\x13C;C\x88\xf6\xcb\xd8\xf6[2\xc9\xe9\xffVXG\xe3/\xe2`'\x9ed\xbe\x1f\xaf\xaf~\x1f\x05\xefċl\xf5H\x02\x1etd\xd8\xe58\x93\x98\xbd\x1f-N\xa9c&c\xed\xd7<+3t\xacʤb\xc3\xf6䩄\xed$\aƭ\x18VY`\x06\x81A\xcb4I\xee\x01\xf7\xcb\x10\xe25\x13\x14\x9f)\x04\xf7}\x0e`Z7\"\x1b\x8ac \x8fIh\xe4\x04\x15ڬ\xb2\xc7\ue795ð\xafsF\n\x1f\aK\x93\f\xcet\x96\\\x9d\xb3\xd4Q\xbfi\x8e\x16e\xd7Ρ,\xe1Ai\xc12\xe3\x06\xad\x13ef\xe2\xd5<\xd38!\xac\xc0\xcb3<\x88-\xe8L\xf1\x12E\x112\xbd\xbe\x80\xf1]\xc7\\\x85p\xbc<8\n\x91*t\xca[\xc7\x10\x97\xf9Rr\xb2\x86J\xabɐV|1ed\xa6\xf3\x98&G\x9d\xd1!\xd2y}\xed\x1b\xdeϨ\xb0C#?\xf24\xf8S\x97\xda\xfbTL\xbc\xb4\xc6.\x15\xe5\xe9㧕\xd3⽜\xef\xf0\xed,ã\xba\x8b\x96\xacw\xd0\xf6\x8fg\xe4\x8ad\x18\x90\v;}\x04#j\xc8}\x12M9\xfe\x96\x89\x069\xa4\xb7\x9d\xe9\x9e\f\xd5!\x95\rn\xc9\xdd\a\xd3K\xa5i\x84\xd7'\xaa5\x82\xf5}\xa2\v{\x82fg\x91\xfb\x9eF\x86\t\xf3\xe4u\xabL\xcb\\\xe8k.\xb3De\xd74l\xd3\xe0\x1a\x9c\xe9\xe6\xd3',\xb1EkYu\xce\x14\u007f\x0e\xabB\xc5\x1e\xb7\x00ۨ\xce\xf5%\xfb\xc8=^بS\xcf\xeb\x1ad\x8b\xe1\xb1:3*VlLڛ\xc6\xef\x19:\x82Ã\xa0G\xb5\xc1|\xd0\u007f\x89O\x00\xf0\x0fZ\xe7\x10Қ\x9c\x81\xf5\xde뤅\xc1\t\xa7|\x83\x8f\x99\xd1\xd9C\xdcp\xf22\x99Lf\xeeGo\rϺ\u007f<\xe8\x1c\v\xe22\xa8U\x93\x8cY9ր\xec\xda\r\x1a\xe2\xc3f\xefЎ\xddy\xae?\xe3\xeb\xba\x03\x1b\a\xfb\x93\xfc\x02\xa5X\xaa\x96L\xfa>*Y\x97S\xc0\x85\xd5\r\xdbg\b\xa7\x8b\xf8܍\x8c\x8b\\\xc0A\x9f\x93Qk4~\xea\xb9}%\x8f\xe9J\xc9#\x95F\xb2g!\xdd_\xffr\"\xd3\x13\xd2a5\t\x0eq\x9e\xd8\xf9\x03\x9d\xf2uN8\x91\xc4Xɴ\xad\x95\xbb\xbe:\xa3\x05w\xfd\xc2d\r\xb3\xe79\xec\xa9EUȉ\xaa\xf7-\xcf2\xd5\xf1\x13\xf09\xa8\xa3\xc5g\xa2P||\xceŠ;\xd4̐\xa5\xfb7\x81\xcb\xe9\xa3\xd5k\xb0\xc27:)\xf3\f\xa9hhCX\nN\x94Z)\x83\x19\x97\t\xf3\xb02\n\"c\xf8\xbfg\xfc\xc8\xea\xc9l\xd0#\xe7\x03ڱY>\x1c\xe96\xfdC\xd0\x1a~\xfdmqHlXI\xc5\x13\xf2\x9b\xe9\x1fYĔ3\xfdՄ\xffY*\x19*\t\xbb\x86O\x9f\x17\xe9\xd9\xf2>\xfd1\x04\r\xfe/\x00\x00\xff\xff\xb0\xddǼ\x99\"\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\x1b\xb9\x11\xbe\xf3Wti\x0f\xcaV\x99\xc3]'\x95\xa4x\xb3\xa5lJɮ\xac2e]\\>\x80\x83\xe6\f\x963\x00\x02`H3[\xfb\xdfS\x8d\a9/\x92\xa2*Z\xcf\xc5\x16\xd0h|\xf8\xd0/4'\xd3\xe9t´xBc\x85\x92s`Z\xe0W\x87\x92\xfe\xb2\xd9\xfa\xef6\x13j\xb6\xf9q\xb2\x16\x92\xcfᦱN\xd5\x1fѪ\xc6\xe4x\x8b+!\x85\x13JNjt\x8c3\xc7\xe6\x13\x00&\xa5r\x8c\x86-\xfd\t\x90+錪*4\xd3\x02e\xb6n\x96\xb8lD\xc5\xd1x\xe5i\xeb\xcd\x0f\xd9߲\x1f&\x00\xb9A\xbf\xfcQ\xd4h\x1d\xab\xf5\x1cdSU\x13\x00\xc9j\x9c\x83V|\xa3\xaa\xa6F\x83\xd6)\x836\xdb`\x85FeBM\xacƜv-\x8cj\xf4\x1c\x0e\x13aqD\x14N\xf3\xa0\xf8\x93\xd7\xf31\xe8\xf1S\x95\xb0\xeeߣ\xd3?\v뼈\xae\x1aê\x11\x1c~\xd6\nY4\x153\xc3\xf9\t\x80͕\xc69\xdc\x13\x14\xcdr\xe4\x13\x80H\x80\x876\x05ƹ\xa7\x94U\x0fFH\x87\xe6\x86T$*\xa7\xc0\xd1\xe6Fh\xe7)\xdb\xeb\x01\xb5\x02W\"m\xe9\xe9fB\nY\xf8\xa1\x00\x01\x9c\x82%BD½2\x80_\xad\x92\x0f̕sȈ\xb8L+\x9eɤ3\xca\x04\xce\xef{\xa3nG\xe7\xb0\xce\bY\x1cC\xf6\u007f\x06\xd5\xc1\xf3\xa0\xf83\x91<\x96\xe8e\x12\x9aFW\x8aq4\xb4y\xc9$\xaf\x10\xc8r\xc1\x19&\xed\n\xcd\x11\x14i\xd9\xe3Nw\x91|J\xfaZ3\x97\xb0s\t\x15A\xb6\xb3\xfdS{\xe8ܾ\x0f\x8a\xc7\x05\x10\x8d\x1a\xacc\xae\xb1`\x9b\xbc\x04f\xe1\x1e\xb7\xb3;\xf9`Ta\xd0\xda\x11\x18^<\xd3%\xb3]\x1c\v?\xf1\xba8V\xca\xd4\xcc\xcdAH\xf7\u05ff\x1c\xc7\x16\x17eN9V\xbd\xdf9\xb4\x1d\xa4\x8f\xfdဖ\x9c\xad\x88\xd7\xffM\xe0.\tҭ\x92]^\xdf\xf7F\xc7\xc0\xb6\x94\xa6@\x9c\r\x82hG뻢\xab\x8f3\x17\x06\xc2\xf4\xe6\xc7\x10\xca\xf2\x12k6\x8f\x92J\xa3|\xf7p\xf7\xf4\xe7Eg\x18@\x1b\xa5\xd18\x91\xa2k\xf8ZY\xa55\n]f\xafIa\x90\x02N\xe9\x04mp\x8a0\x86\xa3f5\xff\xce\xc4\xfck\xaf;X\aN\x17>\x9f\xebN\xdc\x00%;\x10\x16X\\\x1aNq :\x85\xec\x8f\xffX\xdc\xcf\xfe9\xc6\xfc\xfe\x14\xc0\xf2\x1c\xad\xf5\xf9\x1ak\x94\xee\xcd>gs\xb4\xc2 \xa7\xc2\x05\xb3\x9aI\xb1B벸\a\x1a\xfb\xf9\xed\x97q\xf6\x00~R\x06\xf0+\xabu\x85o@\x04\xc6\xf7\xe1/ٌ\xb0\x81\x8e\xbdF\xd8\nW\x8a~\xd2\xda3@\xd6\x15\x8f\xbd\xf5\xc7ul\x8d\xa0\xe2q\x1b\x84J\xacq\x0eW\xbe\x12<\xc0\xfc\x8d\x1c\xeb\xf7\xab#Z\xff\x14\x1c芄\xae\x02\xb8}\xbek{\xe4\x01\xa4+\x99\x03gDQ\xe0\xa1\x10\xed\u007f>xSH\xfc\x1e\x94!\x06\xa4j\xa9\xf0\x8a\xe9\xf6B\x00\xfd\xf9헣\x88\xbb|\x81\x90\x1c\xbf\xc2[\x102p\xa3\x15\xff>\x83Go\x1d;\xe9\xd8W\xda)/\x95\xc5c\xcc*Y\xedB\xb5\xbfA\xb0\xaaF\xd8bUMC\xbd\xc1a\xcbv\xc4B\xba8\xb27\x06\x9a\x19w\xd2ZS\x95\xf1\xf8\xe1\xf6\xc3< #\x83*|\xbc\xa3\xec\xb4\x12T5P\xb9\x10r\x9e\xb7\xc6A\xd2L\x9fm\x82\xf98\x05y\xc9d\x81\xe1\xbc\b\xab\x86\xb2Pv\xfd\x12?\x1e\xa6\xfe\xf4\x8d\x94\x00\xfd\xc0\xf1͒\xe83\x0f\xe7+\xd5g\x1c\xae\xfd\xd6:y\xb8u\xb3D#ѡ?\x1fW\xb9\xa5\xa3娝\x9d\xa9\r\x9a\x8d\xc0\xedl\xab\xccZ\xc8bJ\xa69\r6`g\xfe\xc9<\xfb\xce\xff\xf3\xe2\xb3\xf8\xd7\xf5s\x0f\xd4y\xf4\xbf\xe6\xa9h\x1f;{ѡR\xad\xf8\xfc2q5\xacTN0\x11\f\xe0\f\a\xb1\x994\xf22\x8a\xf6\x13*E?B\xaf\x11oE\xe3!\xf6R\xbb\xa2\x874\x95\xbd]\x84\xd3\xf1\xf7^OF+>\xe9\x93\xd6v\xc9\xde\xe4\xc1\xa1\xfa\x13][\xed\xcdv\x9a\x9c\xed\xd3\f\x9fʾ\x83v\xc9c9t\xed\"\xef!f\xbb\xd4ˣ\aˋ\x9f˹\xa2\xc7@\xf7W\x8b\xd36p3\\\xe1{S\x86G\x9f\x105\xfa7hh8n\x99M\x9b\x8c\xdd7\xb4\xf4\x85\xa5>O\x92:\xe4\xbeT\xa7\x97Ċ\x89\n9\xec\u007f7\xf1\xcdq\xeb\x9b4\xd7c\x95iR\xd4X\xe4>n\x8c\x80\x1e\xaeK}O\xce\x1cNI\xc5@B6UŖ\x15\xce\xc1\x99f8}½j\xb4\x96\x15\xe7\xfc\xeb\x97 \x15^\xf1q\t\xb0\xa5j\xdc\xfe\x19\x1f\x1d-Rqm\xa3\x15\\\xd6J(\x99=\a\xe5\x81d\xc6,n\xef\xf2\xa7M\x0eN\x84\xb2{\u070e\x8c\x0e\xfa\xd0\xedɛdB#s?y븈\x80\xb8\xd19\x0e\xa2\x18\x94\xaaJ֭\x1c%\xa5\xa6^\xa2!\"|\xf3;1\x92\x02\xc7X_Ŀ\xa7\x0eL\x1e4\xa4X\x18T\xc5\x17bΤo\x13\x92\xfd:\x05\\X]\xb1݈\xdet\x12_2\x91\xf9\x92\x1f\x1d,&y!\xb9\xbf\x9f\xbb\xb4\x9f\xb3o\xee\x8f\x17tc?\x15\x8c\xddB\xbb\xefߛ\xdf\xff\xaa\xf1:;\x9c(\xe2\xacc\xc6=7\xec-:\xc2\xe7\"\x9eW=\x1e\xefڡk\x18\xa8\xba\xdb\xfc\x911j\x94\xa8\xc1\xa0G\xce[\xbac/\xb4=\xd2,\xf7\x9d\xfe9\xfc\xf6\xfb\xe4\x90\xeeXNU;\xf2\xfb\xfeOڱVI\xbfP\xfb?s%\xc3O\xcav\x0e\x9f\xbfL 6M\x9f\xd2\xcf\xce4\xf8\xbf\x00\x00\x00\xff\xffe\xe5\xd5&\b \x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc<\xcbr۸\x96{\u007f\xc5)\xcf\"3U\x96ܩY̔w\x19wR\xed\xea\xee\xc4e\xa7ҋ\xa9\xbb\x80\xc8#\tm\x10`\xe3!G\xf7\xd6\xfd\xf7[x\xf1!\x82$$[\xe9\xf4\xe5N\x14p\x80\xf3\xc0y\x13\x17\x8b\xc5\xe2\x82\xd4\xf4\vJE\x05\xbf\x01RS\xfc\xaa\x91\xdb_j\xf9\xf4\xbfjI\xc5\xf5\xee\xed\xc5\x13\xe5\xe5\r\xdc\x1a\xa5E\xf5\x80J\x18Y\xe0\x8f\xb8\xa6\x9cj*\xf8E\x85\x9a\x94D\x93\x9b\v\x00¹\xd0ľV\xf6'@!\xb8\x96\x821\x94\x8b\r\xf2\xe5\x93Y\xe1\xcaPV\xa2t\xc0\xe3һ\x1f\x96\xff\xb3\xfc\xe1\x02\xa0\x90\xe8\xa6\u007f\xa6\x15*M\xaa\xfa\x06\xb8a\xec\x02\x80\x93\no@\xa2\xd2B\xa2Z\ue421\x14K*.T\x8d\x85]l#\x85\xa9o\xa0\xfd\xc3\xcf\t\x1b\xf1H<\xf8\xe9\xee\r\xa3J\xff\xdc}\xfb\vU\xda\xfdS3#\tk\x17s/\x15\xe5\x1bÈl^_\x00\xa8B\xd4x\x03\x1f\xed25)\xb0\xbc\x00\b8\xb9e\x17a\u05fb\xb7\x1eD\xb1Ŋ\xf8\xfd\x00\x88\x1a\xf9\xbb\xfb\xbb/\xff\xfd\xd8{\rP\xa2*$\xad\xb5\xa3L\xd8\x1bP\x05\x04\xbe8\xdc\xec\x06\x1c\x13@o\x89\x06\x89\xb5D\x85\\+\xd0[\x04R\u05cc\x16\x8e\x88\rD\x00\xb1nf)XKQ\xb5\xd0V\xa4x25h\x01\x044\x91\x1b\xd4\xf0\xb3Y\xa1\xe4\xa8QA\xc1\x8c\xd2(\x97\r\xacZ\x8a\x1a\xa5\xa6\x91\xb0\xfe\xe9\xc8Q\xe7\xed\x01.o,\xba~\x14\x94V\x80\xd0o9\x90\f\xcb@!\xbb[\xbd\xa5\xaaE\xed\x10\x9d\x80\x12\xe1 V\xbfc\xa1\x97\xf0\x88҂\x01\xb5\x15\x86\x95V\xeev(-q\n\xb1\xe1\xf4\xef\rle\x11\xb5\x8b2\xa21\xf0\xbb}(\xd7(9a\xb0#\xcc\xe0\x15\x10^BE\xf6 Ѯ\x02\x86w\xe0\xb9!j\t\xbf:\xf6\U00035e01\xadֵ\xba\xb9\xbe\xdeP\x1d\xcfO!\xaa\xcap\xaa\xf7\xd7\xee(Е\xd1B\xaa\xeb\x12wȮ\x15\xdd,\x88,\xb6Tc\xa1\x8d\xc4kRӅ\xdb:wghY\x95\xffѰ\xedMo\xafzo%OiI\xf9\xa6\xf3\x87\x13\xf3\t\x0eX\x81\xf7\xb2\xe4\xa7z,ZB\xdbW\x96:\x0f\xef\x1f?w匪C\xea;\xbaw\x84\xafe\x81%\x18\xe5k\x94\x9e\x89N\xda,L\xe4e-(\xd7\xeeG\xc1(\xf2C\xf2+\xb3\xaa\xa8\xb6|\xffà\xb2\x02-\x96p\xeb\x94\n\xac\x10L]\x12\x8d\xe5\x12\xee8ܒ\n\xd9-Qxv\x06XJ\xab\x85%l\x1e\v\xba\xfa\xf0p\xb0\xa7Z珨\xbcF\xf8\x15N\xffc\x8dE\xef\xc4\xd8it\x1d\x8e9\xac\x85\xec)\a;e\xd9\x03\x9a>\xb4\xf6\xf1\xa7\xdfj\xb0\xc3\u007f\x0e\xb6\xf2\u007f\xcd@+?v\x13\x86\xd3?\f:\x15\xe7O,\x0eT\xca\x00$\xc4\xfd9\xb1X\x0e\xfe\x1f\xa1\xa9}\xf0k\xc1L\x89e\xa3m\a\xb8\x1c\xec\xf8\xfd`\x823G\x84r+\xffV\xfd\xdbm\xf3\xf6_\xabN\x13;&\x12\xc1J \xe5\x1e\x1eP\xee\x90MR\xda>Tc\x95\xd8\xdc$v\xe0\xec\x1cY1\xbc\x01-\r\x8eP\x86HI\xf6#\x84\x89\xb69\x97.\xcd\xf8\xa0\x10\x18-\xb0k(q\f\x12\xfa\x86y\x06.\xb8\xf8\x95J\xac|\\\xfc\xd9Q\xb3}\xe3\x02\x8aw\x1f\u007f\xc4r\x8a<\x90'y\x03D\xde\x1dl\xb6\xbbtp\xf4s\xd1\b\xaeO\x134\xf9\x8c\xc7\x15\x10x½\xf7X\b\a\xcb\x1c\xa2\x9d\xbf\x9b\f\x9f\x86\xc4q\xa9\x17\xef\x1e\xe3ށ\t\xb9\x94\xd9ٹ\xa2\xe0\x9f'L\xf8\xfb\xa9\xa7G@\xbb\xa7\x10\xe1zJ\xda\x17\x8e\x10.\xf2\xce'\x1e\xb8\xbcX\xd4E\xf3\xc8A\xbe\"\x89O\xa4\xfd\th6l\xeb\xe4\x0f\x1dc\xdf(\xcf\"{\n\xb6\xb4\xceDԥ\x0f\x15\xba\xd3\x123c_\b\xa3e\xb3\x90\x97\xfb;>\xee\r\xf7\x9f\x8fB\xdf\xf1+\x1f\x92)'%?\nT\x1f\x85vo\xceBN\xbf\xf1\x13\x88\xe9'\xba\xe3Žڶt\xe8\xa6\xd82\x84\xdb?w>\x93Ұ\x87*\xb8\xe36p\t\xf4p\tS\xbfܴ}\xe8?\x95Q.\x87\xc6\x05_8S\xb9L\xad䉝\tR\xc8\x1eG\x86[k\x16\xf5\vf\x82\xfdl-\x89\x9f\xefS\xc0\x8c\x14X\xc6h\xd3%.\x89\xc6\r-\xa0B\xb9\x992\x1cݧ\xb6\xfa=o\v\x99Z\xd7?GJX\x9ei\x8fOP\xdd\xe5\xfcf\x16\xf6\xe4f\x8c\x8a̞\x1d:\x92\xaf\x1c\x1f:\x8f\x913\xb1\xce\xff\x98\xa5.)KW\\\"\xec\xfe\b\x8d\u007f\x04/\x86\xb6\xdfo\xcc[Ȋ\xd4\xf6\xfc\xfeÚ9'\xd0\xff\x84\x9aP\x99q\x86߹:\x11\xc3\xdeܐ\x19\xeb.cW\xa0\n,\u007fw\x84\r3\xe1\t\xe4\x84\xd5-ȼ!\x17\xeb\x81\xc7r\x05\xcf[\xa1\xbcM]Sd\xa9\x94M\xff\xa1\n.\x9fp\u007fy5\xd0\x03\x97w\xfc\xd2\x1b\xf8\xa3\xd5M\xe3-\b\xce\xf6p\xe9\xe6^\xbe\xc4\tʔĬa<\x99\xe7n\x9f\x9eXts\xddm\x92;\xb8\xb9S\xbbΒ\xc3Z(\xfdS:a7\xb2\x9f\xfb8\xa3\xef\x9b&\xf2^\xb3>{\xc8a5J\xd5zrk\x8d2$\U0007c88d\x11\xc0\vc\xa3\xb9$]\x93\xa0#Mf\xd5\x12xF*|\xcd#g\x8b\xc7x\x8d\x96.G\xfa\xdb\xef\xbfvr\x8c\xf6\x84\xda\xdf]D^۫-DU\x91\xc3*_\xd6Vo\xfd\xcc(\xd3\x01\x90\xe7\xbe\xdc\x18w.\xf3ݽ(C\xae\xbe\xf7L\xf5\x96r \xf1\xf8\xa3\f\x02E\xa0\x16\xf3\x9a\xc8?[\xa2`\x85ț\xdc\xf8\xf7`\xaf+\xca\xef\xdc\x02\xf0\xf6\xd5\xed;\xb4\xe4:\x89\x9d\x91\xd4\rC\x9b\x17\xce\xe2\xe4\xbaF\xa2\x84\xe7-J\xecI\xc50\xe1m=\xc6L\x90\\\xe8n^\xc1\u00adE\xf9F\xc1\x9aJ\xa5\xbb\x1b\xcd\x158\xa3r\xc5\xe1H\x0e[\xec>\xd3\n\x85\xd1'\xf0\xe0};\xbbW\xa0\xad\xc8WZ\x99\nH%L\x86q\xf7\x8f\xb5/\xb4j\xaa\xa8\x81\x03τꦞ\xe42,ZX.\xd5\fu.\x8bW\xb8\xb6\xea\xa8\x10\\\xd1\x12e\xac\xf2{\xceRa\x0f\xee\x9aPfR\xe5\x9b\xd4sl\x98\xca\xdfKyR\x94\xfa\xc9\xcf\xecd\r\xb7\xe2\xb9O\xa0l\x12l\xc9\x0e\x81\xae\x81j@^X\xbe\xa0\xf4*\xdb-\x11\x88\xe1H\x93-\x96y\n\xde>\xc8M\x95G\x80\x85;ٔO&ź\xc3?\x10\xca\xce\xc16+y\xa7\x1f\x8d\xdf\xda\xd9\xdf\xe4h4J%߄\xad\x10\x1e\x90\x94\xfbx>\x88\xd66Tu2 @\x1a\xdeՈg8\x19\xc7\xc4wa\x17\xaf\x19\xb8QN3\x18{\x90ϧ\xba\xeb\xedX\x10g\xf5v\xec\x02\x8d\xa1;%5s\xd7\x03`Met\x9c\xdd\xde\x1b\xa99\xc2\xf3Y\xa1\rP\xb1\xf4I/k>\x83\x1f\xed{\x97F\xca\xe0I\xec\x8ew]\xb28\u06dd\x90\x9f%\xfa\xbah\xdb\x15\x16.)(w\xb80\xfc\x89\x8bg\xbep1\xa5\x9a\xcd\xd67\x8b\x9f\xac8\xbe\xa5\xd2\xe8\x8bW\xbe\bD\xfb{\x06\xa5\x90\xcd\xe6\xa3\x02\xe3))\x98SC\xbe\x8du\xe4\xcf\xd9]L\xad?19\xd4\x1co}\xffino\xd3]zV\xc7\u007fxޢޢ\x8c\x8d\xad\v\xd7ÛR\xabmi\xb2u\x85\x9bf'+?ћ\xf2Mx\a\xedOi_\x99\x1bƮ\xac`\x13ôoE\x95&!DY=@+!\x18\x92ö\u061c\"\xfa\\\xe9\xbc\xdf\x0f֔\xaecC\x98\x88\x8b$0\xf4\xbc\xf4]\x9fݺl\xbf\x06\xee\xb2?q\xa7\u007fz\xabXFy{\xa6\xa8=\xdd@7E\xaf\xa1\xd8t)\xd6\xca`\x18\x17:+\xbf+\xf2\xcd\x14\xa2\xc7\xcb\xcf!ي\x9a\xec\xde.\xfb\xffh\x11\x8a\xd1.\xb3\x90@\xe5y\xdb\xe4\t\x9c\xe5\xe5%\xdd\xd1\xd2\x10֓\xc0\x0e\xcdZ҂\x90\xc0)Kա,\xcd\xe3\xfc\x1e\x8d\xe1S\xed\xf3\xd1G\x9f\xd5iw'\xaff}r\xa5\xba_\x89\x1e\xd1\xe0Ǧf\xf3[\xf2\xf2k\xd1\xd3\xc5\xe3c*Ї\xf5\xe5Q\xa0\xf3u\xe7\x1cOu\xa6\xc6|Be9\xb3\xab\xe8\xc5\t\xe8\x9c\xda\xf1I\x15\xe3\xd9ƛ\xcc:q\xbf\x02<\r\xf2\x88\xeap\x16q\xe6+\xc1G\xd7\u007fC\xbdu\x12\x8f\xec\xaao\xa2\x9e;\tx\xb4\xd6;Uŝ&y\xa2\u009b_\xbb\x9d\x04\xed\xea\xba\xf3\x15\xdb\xd7\xeb\xcbz\r\x17y\\\xd5\xccV]_\xe4Bg\xd4U\x8f\xa9\xa6\xceR\xec\xc4\xcaiS\x19\x1dY\xf7\xd8zi\xbf\x1e:\x024\xa7J:R\x05\x1d\x818Y\x1bͭ}\x8e\xc0\x9e1\xbb\x93R2\xf1g\xe3u\xffJ\xea\x9a\xf2͐\xf3\xb9\xf21)\x1b\x83\xd2iw͞pt\x9d\xe3^X\x91Z\xd2\u007f\x90\x98\bAb҉r-\x96\xf0\x8e\xef\ap]3t\xd2\xe5\xee\u007f\xb1b\xb7\xf5L\x19\xeb~\x95\xe1\xc0vA\x85\x0f\x9cT:\x10\xb6\x03\xc7>fJ2EȞ\xbf;\x17q|:\x18\xdeMcM\xfb\xcf)י\xea\xed\x89\xfese\x98\xa6u\xf2\x10\xd7R\xec\xa8K\x8amq\xdf\xd0\xf3wᾇX\xed\x1d\xa4O\x0f\xcd\xf9Z\x1e\x84\x02$u*\x9e\x911 j\x88~\xe1\xbf\t,\xc4\xc2}\xe6c9\x19\xe5!|;x\xe5\xce`*@\xe5\xf1k\xb5ʂq\xdf\x15\xaaD\x02`ԺL{\xb8\xde\x19w\xef\xfe0(\xf7 v\xae\n\x1a\\\x9e\x99\x86c\xaf)\x94am\x87GP\x80\xfeK\xd4\x03Ͽ\xd5\x18\xf0\x8e{\x1b\x9c\x04{\xb0G\a\xc7*\xad6ڱ\xfa\xd9\x062#C\x93P\xb9hf\xa7\xe5a\xd2\xd4\xe4v\xeb\x9e7\xf69>\xfa\x99\xf5;\xce\x12\x01\x9d\x1e\x03M\x80\xcc\xed\xbe\xcd\xcb\xd8\xcfv۞+\x16\x9a\x8b\x86\xb2\xdd\xc0\xbcn\xdast\xd1\x1e\xd1={DTt\\\\\x94M\xa6\x9c.ٳDGg\x8c\x8f\xce\x11!\x9d\x16#̀<\xe8~\xcd\xe9k\xcd*2e\x97(r\x8aJ\xf3u\xcd\xe9~Ռ>Ռ\xe2\xc7\xdcN3\xfaQ\x8f\xebC͠ᙢ\xa73\xc5O爠\xce\x1bC\xcdFQ\xb3\x923\xf9\xf7\xc99\xf2XM\xfd(J\xbc\x17R\xcf9\xfc\xf7\x87\xe3\x13\x15\xacN\x10$X\t<\x0eM \xe5|\xf9\xe0ǟ\x86T\xba\xd8\x14ֿ\xff2\x87\xcfC3p\x1a\x11\xeb\x92\xc6\xf8,\x81\x87\x9d\xefpQ\x9c\xd4j\x9b\b\xef^\x8ẹ&\xdad\xe2\xe3\xc7\xf6P\xa2ŶS\xb6y\xc6X=\x94\x9d\xab\x87\x0e\xf6d\x1d\x1e\x0fȝ'g\v9eW\x1d\x0f\xfb\xdbT#2\xbfl>\xf9\x9bfO\x9e\x91s\xe9BF\x1b\xd5EYh\xe9rB\x1dbV\x17g|~8m@2?\x8a=\xf9s\xd8yb%\b5\xf6%l\xce\u05ee\u007f*='Ԯ*\xb6X\x1a\x86\x19\x97\xd4\xcdB\xa6o`\x92r>U\xe9\xfau\x83?\xeb\x9al\x1d{\x19\v]\xb6\x15*E6\xf1Z\xa1g\x94\b\x1b\xe4\x96\xc4Sw̴\x8d\xca\xe1\x047\xfd\x12\x96Z\xa4І\x84\x05\xbc\xa5l\x92\xb8\xa9\x04\xa0\xbf\xb9\xcc\x0e!\x9b\xd1sC\xb9\xc6\xcd }\x1a\x9a\xa4\x1f\x90\xa8Û\xee\x06\x84\xf8\xd0\x1d\x1b\x82_O\x03\xffU5q<\xf5\x17\xa3i*q\xcaC\x10n壮˪\xb7Dͩ\xcb{;\xa6\xf9z\xa0s(\x1bM\xf90\xb2\xa7t7\xf3\x02>\xe2s\xe2\xed\a'\xf4.\xa1\x91>J\v\xb8\xe3\xf7Rll\x8c\x91\xf8\xf37B5\xe5\x9b\x0fB\xde3\xb3\xa1\xfcS<\x93\xc7\r\xbe'RS\xc2\xd8\xde\xef'1\xf76\x1e\xe6\xc4\u007f\xf3\xb3G\xfe\x98bR\xc0y6\x1a\xf0\xc3\xda\xe0\x88r\u007f\xd0]\xef\xfeJ\x18\xdd=\x15oT{`ҙY\am\t\x1f\x85Ƙt\xa3}\xa0T\xc1\n\x95^\xe0z-\xa4\xf6\xc1\xd8b\x01t\x1d\x14u*\xc8 \x949_\xc3\xdf\xd3g\x1d\x90\xa6\xf0\xdbX>!\x81\xf0=Hw*\x9c\x93R\x91\xbdo\xad#Ea\xac\x1e\xb8V\x9a\xa4\fڋ\\[\xe7\xdc\x04i\x1e\xc9K\xf4=\xb5\xee\xf8\xe6+@S\xadP\xba\xdea\xfb\xb7'\x9d\xfb\xac\xc0\xab\xa0d\xc1\x01\xdc\xc7\a\x9d\xaf\x9a@\xd9㜎\x8f\xa7\x94\x8f\xfb_h\xc2\xee\xc6\x1d\xb5~#m38\"\xe0\xa6\x0f\xd1\xe8]H6\xde&DU\x9cjyVl\t\xdfX\xf1\x91\xc2l\xb6Q\x04\xc74\xf5X>ĸ\xfb\xd0jwRUL^k#y'\xf7\x12\xd2\xd9e\xbb\xdd)\xa0\xd3$\x9c\xf23{\xa6u\xce\xd3\xec\r~\x99\v\xe1\x16\xb6\x0e\xc4\xf7k\xfaw\x8d\xee~\x9f\xe3\x04|9\x18~\xd0 i݁\x16b0\xdc\t\xe2\xfc']\xc7\xfblW\f\xffk0\xe2\x1b7:>\x13\xc9)\xdf\xcc!\xff[\x18\x96\xf0\x81\x02\x84\x84\x17\x94@\xa2\xf1\x8b\x8e\xf2\x82\xe2&G\xaell<\xa3\x17\xf8A\xc934x\xe9\x04\xb9\xec\x109\xac\x14\u07b4\xf1\x03)\n\xacuh@\xee\xde\xd6|y\xe9~\xc4\xeb\x98\xdd\xcfBp\xaf\x16\xd4\r\xfc\xff\xdf.\"B_\xe2\xad\xcb\xf6\xe5\xbf\x02\x00\x00\xff\xff鐱=\xdaZ\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c7l\xef\xfa\x15\x18\xf5\xc1\xed\x8c\xee\x14O\x1f\xdaћ++SM\\\xdbc)zi\xfb\xc0\xdb\xc5\xe9\x18\xed\x92\x1b\x92{\xf25\x93\xff\xde\x01\xc8\xfd\xbc\xfd\xe0\x9e\xa5i\x9a\x11_\x12\xed\x91 \b\x80\xf8 A\xf8l\xb5Z\x9d\x89B>\xa0\xb1R\xab+\x10\x85\xc4\xef\x0e\x15\xfde\xd7O\xffj\xd7R_\xeeߟ=I\x95^\xc1ui\x9dο\xa1եI\xf0#n\xa5\x92Nju\x96\xa3\x13\xa9p\xe2\xea\f@(\xa5\x9d\xa0ϖ\xfe\x04H\xb4rFg\x19\x9a\xd5#\xaa\xf5S\xb9\xc1M)\xb3\x14\r\x03\xaf\xa6\xde\xff\xb4\xfe\x97\xf5Og\x00\x89A\x1e~/s\xb4N\xe4\xc5\x15\xa82\xcb\xce\x00\x94\xc8\xf1\nl\xb2ô\xccЮ\xf7\x98\xa1\xd1k\xa9\xcfl\x81\t\xcd\xf6htY\\A\xf3\x83\x1f\x140\xf1\xab\xb8\v\xe3\xf9S&\xad\xfb\xa5\xf3\xf9\x93\xb4\x8e\u007f*\xb2҈\xac5\x1f\u007f\xb5R=\x96\x990\xcd\xf73\x00\x9b\xe8\x02\xaf\xe03MU\x88\x04\xd33\x80\xb00\x9ez\x05\"M\x99T\"\xfbj\xa4rh\xaeuV\xe6\x15\x89V\x90\xa2M\x8c,\x1c\x93\xe2\xce\tWZ\xd0[p;l\xcfC\xed7\xab\xd5W\xe1vW\xb0\xb6\xdco]섭~\xf5$\xf2\x00\xc2'w ܬ3R=\x0e\xcd\xf6\x01\xae\x8dV\x80\xdf\v\x83\x96P\x86\x949\xab\x1e\xe1y\x87\n\x9c\x06S*F\xe5\xdfD\xf2T\x16\x03\x88\x14\x98\xac{x\x06L\xba\x1f\xe7p\xb9\xdf!d\xc2:p2G\x10aBx\x16\x96q\xd8j\x03n'\xedh\x1ao`%2؋\xac\xc4\v\x10*\x85\\\x1c\xc0 \xcd\x02\xa5j\xc1\xe3.v\r\xff\xa1\r\x82T[}\x05;\xe7\n{uy\xf9(]\xa5b\x13\x9d祒\xeep\xc9\xdaRnJ\xa7\x8d\xbdLq\x8f٥\x95\x8f+a\x92\x9dt\x98\xb8\xd2\xe0\xa5(\xe4\x8aQW\xacf\xd7y\xfa\x0f\x15G\xed\xbb\x0e\xaeG\xfb\xcd7V\x84\x13\x1c \x8d\xe8\x05\xc6\x0f\xf5\xabh\bM\x9f\x88:\xdfn\xee\xee\xdb\xc2$m\x9f\xfaL\xf7\x96\x845, \x82I\xb5Ű\xa3\xb7F\xe7\f\x13UZh\xa9\x1c\xff\x91d\x12U\x9f\xfc\xb6\xdc\xe4\xd2\x11\xdf\u007f/\xd1:\xe2\xd5\x1a\xae\xd9\xee\x90\x1c\x96\x05\xed\xc0t\r\xb7\n\xaeE\x8eٵ\xb0\xf8\xea\f J\xdb\x15\x116\x8e\x05m\x93\xd9\xef\xec\xa9\xd6\xfa\xa12o#\xfc\xaa\xf6\xf8]\x81Ig\xcb\xd08\xb9\x95\to\f֞\xb5\n\xe8iP߆w-\xff\xc2j\xaa\xff\xb5\x87\x87\xd7eլh\xc9~\xb8\x1ds\xb81c$W\x1e\x1a\xe9\x14\xa5\xfb\xdc\x1d҂-J\x04(3\x98t\xb5^\xac};\x82\tAխGp<\xe2*\xff\x84yAjc\x06\xc5\xfbЍP$\xfa\xa4\xb5;U\x19\xfeJ\xcd\xea\xa0]\xe1H\xb9\xf1t;$\xbe\xede\x1a\xb4\xd7\x11Wa\x92\xb3\xd4\x12+\xef\x94(\xecN;\xb2q\xbatC\xbdz\v\xb8\xbe\xbb\xed\rjq\x9e\xb0b\x1bΌv\x1a\x9e\x85<\xe6\xb4o$\x97\xd7w\xb7\xf0@.\x11V0\xc1[rp\xa5Q\xac\x8e\xbf\xa1H\x0f\xf7\xfaW\x8b\x90\x96\xac\x95*\xbb|1\x02x\x83[\xda\xf4\x06\t\x06\r@ch\x0fXFM\x97n\xcd\x0eG\x8a[Qf.(9i\xe1\xfdO\x90KU:<\xe6;L\xf3\xde\x13\x89\xc1\xf9\xd5\xd8{\xfd\xb3\xf5\x8c\x8c \xe9Ǒ\xa1\x03[\xaa\xd0)\xec\xb9\xdf\x18Ue\x86`\x0f\xd6a\x0e\x9b\x00\xa5\xb6\xd5\xcc\x15\xd6\aY\x16\xc0X\xd8\x1c*܇\xd7M^\xb8\xd8dx\x05Δ\xc3\xd3Nm\xdd!\xda|C\xebd\x12A\x99\xf3>i\xfc\xc8\x01\xc2\x18\xfea\x84(=\n\x90\x91\x17O\xe4h\x06\n\x91\xb7\x90e-\xe2\xceS\x05\xe0\xbf\x14|$\x03\x97\x90ٹ\n\xe6Lb\xc6&TiȴzD\xe3g$W\xe1Yf\x19oi\xcc\xf5\xbe\xe3d\xb5\x1b\xd9\x16\x83\x19\x19Iؖdv\xd6@\xb2?*#RY\x87\"]\x9f\xbf\x16\xf3\xf0{\x92\x95)\xa6u\x983\xa8Kz\x8c\xbb9\x1a\xc4\x01\xa1\x90\x8a43\x85_DtU\xff:B\r\xf65\x85A2\x18 \x95\x87I\xa4!E\xb3\x19Q\xd2Ԥ\xc3|\x04\xcfٝ\xbc\x80j\xc2\x18q\x98\xa0Y\x154/!Y=&\xb8b\x99L\x90\x88U;\\L5&\xcd\xc8\xfa\xfe\x1f\x12l\xa7\xf5S\f\x91\xfe\x9d\xfa5\x8e%$|6\x01\x1b܉\xbd\xd4\xc6\xf6\xa3\x13\xfc\x8eI\xe9Fw\x9bp\x90\xca\xed\x16\r\xc1›\x8e\xbf\xa7\x885mV\xa9\x99i\xc6\x1f\xad\xaba:1\x8f\xa91\xb6\x14v_F\xa1\x02#NV\x8fuC*\xf72-E\xc6jB\xa8įO\xd4\xf8\x8d)\xb7\x19\x818\xc2\xdf+\xa3j\x15ĥ\x8eW\xaa\x15\x92ۗk3f\xb7|;\x063N\x86\x8d`gr̅k\x9a)3\xb4\x01\x15o\xfe\x1a\xbds\xd1p\xca\at\x99\xd8`\x06\x163L\x9c6\xe3\xe4\x89\x11\x02\xdfb\xf5\xe7\be\a4i\xd7ߚU\xa2M#\x87l'\x93\x9d7V$e\f\vR\x8d\x965\x86(\x8a\xec0\xb5h\x88\x91\x8c0ٜ\xd2hZ\x84\xfa\xe8\xc3\x1dS$M\x8b\xd4\xc1M\x9b\xd1\xc6]\xaa\xd7b\xf3F\xf4\x0e\x9aꇄ\xfd\xf6h\xf8\xcb\v;\x91[\xa2]\xc3\xed\x160/\xdc\xe1\x02\xa4\xab\xbe\xc6@%W\xb1\xc1\xe3oƸ\xd3v\xcbm\u007f\xf4\x8b\xef\x96\x17\xe1Z\x8d\xc6߄il\xac\ue0adZİO\xed\x91\x17 \xb75\xc3\xd2\v\x8a!\x1d\xb2/5\x87h\xcbљ\xe5\xdcK\x12(\xd6\xf6R˅Kv7\xf51PĈ\x1e\xad\xfa\x00\xbc_^\xc50̃\b\x90P;\x15|\x82)\r\xe6\xfed\xf4\x9e\xf7G\xf3\x85=\xc0\x0f\x9f?b:G2\x88\x97ԣE}\xe8y:m\x14x\x81Q [\x8bb7\xad\x8e\xf1\xfc\xf9\xf7\x05\bx\u0083\xf7\xac\x06\x83ˡF\xac\x155H\x83|\x18\xcfj\xe4\t\x0f\f*\x9c\xaeG\xc1[\"*\xbe=\xe1!\xb6k\x8f\xa8\x84_8\xd7\xf3ԥ\x0f\xbc\x8a\x98\xadԴ\x9a\xa8a\xef\x80\xd3q\x8b\x85eJ\xa9j\x15\xc5O\\vͰΕ\xd2\x13\x1e\xdeY\xcf>\xda5;Y,\xa0\x00)l\xb0\xc8;\xac\xbaKy\x10\x99L\xeb\xc9x\x9f,\x80x\xab.\xe0\xb3v\xf4\x9f\x9b\xef\xd2\x12\x8a*\x85\x8f\x1a\xedg\xed\xf8˫\x92\xd8/\xe2D\x02\xfb\xc1\xbc-\x957\vD\x97E\xf378\xb0\t%\x11\xad\xd9&-\xdc*\x8a\xcf<}\x96\xb0i\x87\x15r\x1e\xad\xbc\xb4|\x1b\xa3\xb4Z\xb1\x99\xaef[\x00\xb4\x8dW`\x956\x1dN],\x848\x88b@\uf7ac\x95\xff\xe5\xe8\x1ek\xaa\x19,2\x91`Z\x9dJ\xf3\xa5\x99p\xf8(\x13\xc8\xd1<\"\x14d7\xe2\x85j\x81&\xf7\xed\x04)\x8cw-\xaa\x16\xcc\xc2\xc0\x1d\xd0P[Ѯ\x8f\xecY\xb19\xaa\xfb\xc8\r\xd9t\xf7\xb8U\xb2yg\u007f(\x8a\xfa픎e\x96e!\xbf\x8e}\x10\x8f\xa4w?r\xc1\xc7\xd6\u007f\x90ye\xf1\xfe3\xce\x1a\ni\xec\x1a>pBK\x86\xed\xf1\xd5)ak\xaa(\x90\x84\x89\xb4@r\xb2\x17\x19\xb9\x0f\xa4\xbc\x15`\xe6\x9d\t\xbd=\xf2\xa0\xe2T\xcc\xf3N[o\xf3\xebc\xf5\xf3'<\x9c_\x1ci\xaf\xf3[u\x1e\a\x93t\xfe\x91Ҫ\xbd\x16\xad\xb2\x03\x9c\xf3o\xe7\xec\x98-\xd9\"'8o\v\xa4:\xba+\xa7\x8e,\t\x05(֮\xbc\x16\x1a\\'X\x90\v?\xb7\x8ah\x99.\xb4\x1d\xb9]\x1cA뫶\xce\x1f\x00v\xdc\xed\x81\x13\u0098\xe8/\x9c\x1a\x82\xd8:4`\x9d6U2\x03\xa9\xdd\xde\x019q\xde\xce\xf3\x9eX]\x9fFz\xc0\x14d\x9e7\x1a\xc2\xeb\xf4s\x9f\xe5@\xff?\x0f3ag\x89a\x17F'h\xed\xbc(EZ\x8e\x99\x03\xdb\xfa\xb0V\xf8\xe0m\x1b\xa5\x9ac\x8e\x92\xab\xb6\xcc\x15'Ҟ\x10\xd8\xdc|o\x9d;\x93\x1a\xa2\xbfcD\xf9\x14\x1c\x81\x13\x1d\xf3\\\xf4\x13k\xa2ѽ\xf6\xa3\xab\r\x18\x80\xf9\x80\xc9<\x96\xacT\x96\xf9\xcdA$\xffj\x8eG.\xd5-O\x04\xef_\xcdY\x81J\x95㩡\xccu5\xbeaH\xfd!6~\x85*=C\xf3]\x8d\xc1\x0eg\x8fo2\xe29\x05\xe4L+\xedڇ5a\xa6w\x16\xb6\xd2X\xd7 \xbc\x00\xaa\xb4|M\xfd\xba1\xa6\xba1\xe6\xe4\x10\xf3\x8b\x1f\xdd:V\xdc\xe9\xe7\x90Դ$\xb0\xae\x88\xbf\x13{\x04\xb9\x05\xe9\x00U\xa2K\xc5\a^\xa4.h\x9a\x05\x10=\x13\xbd1\x89\xb4\x99\xad\xc1\xaa\xcc\xe3\t\xb2b\xe9\x94j\xf6t\xac=\xe4g!\xe3N\xa7\xe04\xb6\xba\xa9ġ\xa1\xd6͆\n\x19D\xed\xec\xb5\\|\x97y\x99\x83ȉ-K\xe2ƭ\xcf=\xaaR\xdd<\xaf\x9f\x85t!\x83\xd8_\xac.Ӧ\x89\u038b\f\x1dVYE\x89VV\xa6X\xbb\x0f\x81\xff\x839ZcM\xc0VȬ4\vt\xf4b\xce,\x8dۂzz\xf9`,\x1e\x91\x15\x133\xf2\xd0}\x81\xd3\x1f\xa8\xc5\xe7\xa5-\xcdF\xab\xf3\x87\xe6\xad\xca\x0f\xe4\xa0-{ 0\x9bo\x16\x834\xc4d\x99\r\xe7\x8f\xcd@]\x92[\x16\x1b\x83G\xe4\x91\xc5g\x8fő\a\xf8\xb5}l\xceX\xb4\xd7\x16\x9b\x1f\xf6:Ya\x91\xb9`\xad\f\xafY\x90'f\x80E\x13,.\xdb+:ǫ\x95\xb95O\xad\x89̮\xe1|\xadY\x90C\xf9\\1YZQ\xb8F\xe7f\xd5\x19W\xf3'\x89?\x94\x91\xf5\xf2\xb9\xdf/\xe9\xe7O\xe7WEeUE\xc5\x02\xf38G\xe5M-͖\x8a\xa2\xea\xd2̨:\xebib\xe2\xa8|\xa8\xe3\\\xa7\xa9\xa5\xccfA\x8dg8M\x81\x1d\xca}\x8a\xc8k\x9a\x00\xd9\xcexZ\xec\x06\xccJ\xd3L\x87\xe1\x8a\x18U\x9b\xb7\xb5\xd9\xff\x85\x04\xfe袵\xe9\xb8\xc01Qɗ\xde\x10\xe2}\xe5\xf5\r\xb9\xd5\xe31\x9ew\xb6Op\xabG@\xden!/3'\x8b\xacU\x92\xc2\xed\xf0P?y\xffM\xf3\xd3\xcb́\xa1}\xf9V\v\xf0\x18\xc8n\x80 ,\xe8\a]?\xfc\xcc%\xfa\x1a\u007f\xd1]R\xec\x15|\xdc5\xcf\xfc\xf3\x95\xc8g+\x91\x97@1\xd8G>OY\xfe,%\x92\xce'\x06[\x93SG>?Y\x14n\x9d\x18pMB\x9czn2\x1drM\x1f\xa7\xf5\x9f\x99\x9c\xe0NDH\xd8l\x97\x1f\xbe\x11\xd0&E3{\xb9\xb2D4g\x85\xb2\x17\x13u\xe7\xefU\x1d\xa8\xaa{Q\xaf\xf6\xc5\xcd\x18wt\xfd\n>\x81_\xa4J=oH\b[\xfeE\xe7\xf6\xa7q~\xc6\xee\x80\xda\x1eg(\x8b鯍,\x16\x82\x14)\x87E|\xbdm\xd7p#\x92]=\xc3\bH\x9ew',l\xb5Ʌ\x83\xf3\xfa>\xee\xd2O@\u007f\x9f\xaf\x01~\xd6\xf5Eh\xab\xca\xcd\bT+\xf3\";P\xf4\x03\xe7m0?&8\xa3\xc2gC\xb9\xbfP\xcf,\"\xfe\xbd\xeb\x8e\x18*7\x19ʺU\xb0'\xd8,\xd4\x01\xbe>\xb0\x17\xc5ş\x92\xa6HV\xf0\x91\xaa\b\xb8WCk\x04\xe4X\xbd\xc8E\xc4\x1a\xbf\x14\xb6N\x1b\U000487f4/\xe9\x19C\xad\xee\x88NUנ\xa3\xaa\x14\x91\xf0\xe6kdeU1\xe6>\xc0&s\xec\xa8\xca a;\xa6\xbcf\xf6\xb7sY\xc4\xe2\xee\xef?\xf9\x059\x99\xe3\xfac\xe9/\xe8W\x850\x16\x89\xd2\xd5B\xfd\xa0\u0378}\xdb\xe9g.\xd3\u05ee\xbb٪l\x8c\x9c\xa9ƹ\x00'\xadfߩlY\x91.F\xd8\x1f\x86G\xb6\xd4I\x8b\x89SW\xfaz;\nKX\xab\x13\xc9\x1a\x88\x8f\x828A\xec\xf5J\xc4MY\x92\teQZ\xfc\xf2\xac\xd0|\xab6\xaa\xbdUc\x855;$\xfc\xf5h\xe0hQM\xa7Y\xef\xf5\xba\x0f\xd9;\x15\bd}\x11\xd2\xeaLKں\xf4\xec1\xe9f\xf6\xff\xf8\xde\x1f\xf6YW\xc3\xd5^Wu\x01ڳ\b\xca\xfa\"\xab1\xb5\x85}5\xd6D\x14\xae4\xc1\xac&\xa5\xe1zy\x04\x04}9\xb9Ӫ\v7\xd5\xdagx\xd9\xd4oo\xa2\xfc\xd9j\xf1\x03\xfc\xab\xeb\x03\x8f\x16\xcc\xf56\xd5Ws_\x11\xfc\xd3\xd89\xb8\x0f\xb8\xbe\xe0\\-e\xeaS'\xf7\x06B\xf3\xc0\xaa.\xe1\xdd\x18\xea\xc3ٚ+\xf8\x8c\xcf\x03_o\x14-\xe2\xf8.ͧdb\xcag\x03C\x95\xd5'\x97\xb8\xafGq>쀶誹^\xf7^\xa2\rW\xa9\xad\xbb\xf8\xdc\xd7!\xb6\xfe\xa3\xdc\xfa\x83\x9b\x84\xd6\xf4OG=F\x15פ\xd2\x1aSX\x83[\xea\xe8\xa3E\xb3粰\x95\x90\x04\x1b\xde\xfeRn\x9a2\x91\xf0ǟgͮ\x14I\x82\x85\v\t]\xed\u007f\xc5\xe2ܗy\xad\xfe\x91\n\xfe3\xd1\xca;\xd8\xf6\n\xfe\xf3\xbf\xcf \x18\xe0\x87\xea_\xa2\xa0\x8f\xff\x1b\x00\x00\xff\xff\t\xb7x\x1e\xf3c\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=Msܸrw\xfd\x8a.\xe5\xb0I\x95f\xbc\xae\x1c\x92\xd2͑\xbd\x15\xd5sl\x97\xa5\xe7K\x92\x03\x86\xec\xd1`E\x02|\x008\xf2\xe4\xd5\xfb\xef\xa9n\x00\xfc\x1ar\b\x8e\xa4\xca\xe6\x95p\xd9\x15\ah4\xba\x1b\xfd\x014\xda\x17\xab\xd5\xeaBT\xf2\a\x1a+\xb5\xba\x06QI\xfc\xe9P\xd1_v\xfd\xf8\xafv-\xf5\xbb\xfd\xfb\x8bG\xa9\xf2k\xb8\xa9\xad\xd3\xe5w\xb4\xba6\x19~ĭT\xd2I\xad.Jt\"\x17N\\_\x00\b\xa5\xb4\x13\xf4\xd9ҟ\x00\x99V\xce\xe8\xa2@\xb3z@\xb5~\xac7\xb8\xa9e\x91\xa3a\xe0q\xea\xfd\xaf\xeb\u007fY\xffz\x01\x90\x19\xe4\xe1\xf7\xb2D\xebDY]\x83\xaa\x8b\xe2\x02@\x89\x12\xaf\xc1f;\xcc\xeb\x02\xedz\x8f\x05\x1a\xbd\x96\xfa\xc2V\x98\xd1l\x0fF\xd7\xd55\xb4?\xf8A\x01\x13\xbf\x8a\xbb0\x9e?\x15Һ?\xf5>\u007f\x96\xd6\xf1OUQ\x1bQt\xe6\xe3\xafV\xaa\x87\xba\x10\xa6\xfd~\x01`3]\xe15|\xa1\xa9*\x91a~\x01\x10\x16\xc6S\xaf@\xe49\x93J\x14ߌT\x0e͍.\xea2\x92h\x059\xda\xcc\xc8\xca1)\xee\x9cp\xb5\x05\xbd\x05\xb7\xc3\xee<\xd4~\xb7Z}\x13nw\rk\xcb\xfd\xd6\xd5N\xd8\xf8\xab'\x91\a\x10>\xb9\x03\xe1f\x9d\x91\xeaal\xb6\x0fpc\xb4\x02\xfcY\x19\xb4\x842\xe4\xccY\xf5\x00O;T\xe04\x98Z1*\xff&\xb2Ǻ\x1aA\xa4\xc2l=\xc03`\xd2\xff8\x87\xcb\xfd\x0e\xa1\x10ց\x93%\x82\b\x13\u0093\xb0\x8c\xc3V\x1bp;i\xe7iB@z\xd8zt>\x0f?{\x84r\xe10\xa0\xd3\x01\x15\xa5z}$\x91=\x98\x1f\x1e0\x01\x18\x93\xa8\x12\xb5e\xe1hG\u007f\xeb~\xf2\x006Z\x17(\xd4E\xdbi\xff\xde\xcb^\xb6\xc3R\\\x87κB\xf5\xe1\xdb\xed\x8f\u007f\xbe\xeb}\x86\x81,\x05J\x81\xb4 \xe0\ao\f0a\v\x83\xdb\t\a\x06\x89\xf3\xa8\x1c\xf5\xa8\f\xae\"u\xf3\x06$\x806P\xa1\x91:\x97Y\xe4\n\x0f\xb6;]\x179l\x90\x18\xb4n\x06TFWh\x9c\x8c[Ϸ\x8e\xaa\xe9|\x1d`\xfc\v-\xca\xf7\U00092216\x85/l(\xcc\x03\x1d\xfc\xfe\x90\xb6ş\x99\xd4\x03\f\xd4I(Л\xdf1sk\xb8CC`\"֙V{4D\x81L?(\xf9?\rlKR\xefX\x18\x1d\x06}\xd06\xde\xc0J\x14\xb0\x17E\x8dW T\x0e\xa58\x80A\x9a\x05jՁ\xc7]\xec\x1a\xfeC\x1b\x04\xa9\xb6\xfa\x1av\xceU\xf6\xfaݻ\a颊\xcdtY\xd6J\xba\xc3;֖rS;m\xec\xbb\x1c\xf7X\xbc\xb3\xf2a%L\xb6\x93\x0e3W\x1b|'*\xb9b\xd4\x15\xab\xd9u\x99\xffC\xe4\xa8\xfd\xa5\x87\xeb\xd1~\xf3\x8d\x15\xe1\t\x0e\x90F\xf4\x02\xe3\x87\xfaU\xb4\x84\xa6OD\x9d\xef\x9f\xee\xee\xbb\xc2$\xed\x90\xfaL\xf7\x8e\x84\xb5, \x82I\xb5Ű\xa3\xb7F\x97\f\x13U^i\xa9\x1c\xff\x91\x15\x12Ր\xfc\xb6ޔ\xd2\x11\xdf\xffR\xa3uī5ܰ\xdd!9\xac+ځ\xf9\x1an\x15܈\x12\x8b\x1ba\xf1\xd5\x19@\x94\xb6+\"l\x1a\v\xba&s\xd8\xd9S\xad\xf3C4o\x13\xfc\x8a{\xfc\xae¬\xb7eh\x9c\xdcʌ7\x06k\xcfF\x05\f4\xa8o㻖\u007fa55\xfc:\xc0\xc3\xeb\xb28+Z\xb2\x1fn\xc7\x1cn\xcd\x18ɕ\x87F:E\xe9!wǴ`\x87\x12\x01\xca\f&}\xad\x97jߎ`BPu\xeb\t\x1c\x8f\xb8\xca?aY\x91ژA\xf1>t#\x14\x89>y\xe3NE\xc3\x1fլ\x0e\xda\x15\x8e\x94\x1bO\xb7C\xe2\xdb^\xe6A{\x1dq\x15Nr\x96Zf\xe5\x9d\x12\x95\xddiG6N\xd7n\xac\xd7`\x017w\xb7\x83A\x1d\xce\x13VlÙ\xd1NÓ\x90ǜ\xf6\x8d\xe4\xf2\xe6\xee\x16~\x90K\x84\x11&xK\x0e\xae6\x8a\xd5\xf1w\x14\xf9\xe1^\xff\xd9\"\xe45k\xa5h\x97\xaf&\x00opK\x9b\xde \xc1\xa0\x01h\f\xed\x01˨\xe9ڭ\xd9\xe1\xc8q+\xea\xc2\x05%'-\xbc\xff\x15J\xa9j\x87\xc7|\x87Ӽ\xf7Dbp~5\xf6^\xfff=#\x13H\xfaqb\xe8Ȗ\xaat\x0e{\xee7EUY \u0603uX\xc2&@il5s\x85\xf5AQ\x040\x166\x87\x88\xfb\xf8\xba\xc9\v\x17\x9b\x02\xaf\xc1\x99z|\xdaS[w\x8c6\xdf\xd1:\x99%P\xe6rH\x1a?r\x840\x86\u007f\x98 ʀ\x02d\xe4\xc5#9\x9a\x81B\xe4-\x14E\x87\xb8\xf3T\x01\xf8/\x05\x1f\xc9\xc0edv\xae\x839\x93X\xb0\tU\x1a\n\xad\x1e\xd0\xf8\x19\xc9Ux\x92E\xc1[\x1aK\xbd\xef9Y\xddF\xb6\xc5`AF\x12\xb65\x99\x9d5\x90\xecOʈT֡\xc8ח\xaf\xc5<\xfc\x99\x15u\x8ey\x13\xe6\x8c\xea\x92\x01\xe3>\x1d\r\xe2\x80PHE\x9a\x99\xc2/\"\xbaj~\x9d\xa0\x06\xfb\x9a\xc2 \x19\f\x90\xca\xc3$Ґ\xa2\xd9L(ij\xd2a9\x81\xe7\xecN^@5a\x8c8\x9c\xa0Y\f\x9a\x97\x90\xac\x19\x13\\\xb1BfH\xc4j\x1c.\xa6\x1a\x93fb}\xff\x0f\t\xb6\xd3\xfa1\x85H\xffN\xfdZ\xc7\x122>\x9b\x80\r\xee\xc4^jc\x87\xd1\t\xfeĬv\x93\xbbM8\xc8\xe5v\x8b\x86`q@\xdd\xc4ߧ\x88uڬR3\xa7\x19\u007f\xb4\xae\x96\xe9\xc4<\xa6\xc6\xd4R\xd8}\x99\x84\n\x8c8Y=\xd6\r\xb9\xdc˼\x16\x05\xab\t\xa12\xbf>\xd1\xe07\xa5\xdcf\x04\xe2\b\u007f\xaf\x8c\xe2*\x88K=\xafT+$\xb7\xaf\xd4f\xcan\xf9v\ff\x9a\f\x1b\xc1\xce\xe4\x94\v\xd76S\x17h\x03*\xde\xfc\xb5z\xe7\xaa\xe5\x94\x0f\xe8\n\xb1\xc1\x02,\x16\x989m\xa6ɓ\"\x04\xbe\xa5\xea\xcf\tʎhҾ\xbf5\xabD\xdbF\x0e\xd9Nf;o\xacH\xca\x18\x16\xe4\x1a-k\fQU\xc5\xe1Ԣ!E2\xc2dsJ\xa3m\t\xeac\bwJ\x91\xb4-Q\a\xb7mF\x1b\xf7\xa9ވ\xcd\x1b\xd1{h\xaag\t\xfb\xed\xd1\xf0\x97\x17v\"\xb7D\xbb\x86\xdb-`Y\xb9\xc3\x15H\x17\xbf\xa6@%W\xb1\xc5\xe3\xef\x8cq\xe7\xed\x96\xdb\xe1\xe8\x17\xdf-/µ\x06\x8d\xbf\x13\xa6\xb1\xb1\xba\v\xb6j\x11\xc3>wG^\x81\xdc6\f˯(\x86tȾ\xd4\x1c\xa2\x1dGg\x96s/I\xa0T\xdbK\xad\x14.\xdb}j\x8e\x81\x12F\fh5\x04\xe0\xfd\xf2\x18\xc30\x0f\x12@B\xe3T\xf0\t\xa64X\xfa\x93\xd1{\xde\x1f\xed\x17\xf6\x00?|\xf9\x88\xf9\x1c\xc9 ]R\x8f\x16\xf5a\xe0\xe9tQ\xe0\x05&\x81\xec,\x8aݴ&\xc6\xf3\xe7\xdfW \xe0\x11\x0f\u07b3\x1a\r.\xc7\x1a\xb1V4 \r\xf2a<\xab\x91G<0\xa8p\xba\x9e\x04o\x89\xa8\xf8\xf6\x88\x87Ԯ\x03\xa2\x12~\xe1\\\xcfS\x97>\xf0*R\xb6R\xdb\x1a\xa2\x86\xbd\x03N\xa7-\x16\x96)\xa5\xd8\"\xc5\xcf\\vðޕ\xd2#\x1e~\xb1\x9e}\xb4kv\xb2Z@\x01R\xd8`\x91wX\xbcK\xf9!\n\x997\x93\xf1>Y\x00\xf1V]\xc1\x17\xed\xe8?\x9f~JK(\xaa\x1c>j\xb4_\xb4\xe3/\xafJb\xbf\x883\t\xec\a\xf3\xb6T\xde,\x10]\x16\xcd\xdf\xe2\xc0&\x94D\xb4a\x9b\xb4p\xab(>\xf3\xf4Y¦\x1dF\xe4qCv\xba{\xda*ټ\xb3?\x94D\xfdnJ\xc72˲\x90_\xc7>\x88Gһ\x1f\xa5\xe0c뿒ye\xf1\xfe[\x9a5\x14\xd2\xd85|\xe0\x84\x96\x02\xbb\xe3\xe3)ag\xaa$\x90\x84\x89\xb4@r\xb2\x17\x05\xb9\x0f\xa4\xbc\x15`\xe1\x9d\t\xbd=\xf2\xa0\xd2T\xcc\xd3N[o\xf3\x9bc\xf5\xcbG<\\^\x1di\xaf\xcb[u\x99\x06\x93t\xfe\x91\xd2j\xbc\x16\xad\x8a\x03\\\xf2o\x97\xec\x98-\xd9\"g8o\v\xa4:\xb9+\xa7\x8e,\t\x05(֎^\v\rn\x12,ȅ\x9f[E\xb2LW\xdaN\xdc.N\xa0\xf5M[\xe7\x0f\x00{\xee\xf6\xc8\taJ\xf4\x17N\rAl\x1d\x1a\xb0N\x9b\x98\xcc@jwp@N\x9c\xb7\xf3\xbc'V7\xa7\x91\x1e0\x05\x99\x97\xad\x86\xf0:\xfd\xd2g9\xd0\xff\xcf\xc3\xcc\xd8Ybؕ\xd1\x19Z;/J\x89\x96c\xe6\xc0\xb69\xac\x15>x\xdb&\xa9攣\xe4ؖ\xb9\xe2D\xda3\x02\x9bO?;\xe7Τ\x86\xe8\xef\x14Q>\aG\xe0DDz\x14\xc3Ědto\xfc\xe8\xb8\x01\x030\x1f0\x99\x87\x9a\x95\xca2\xbf9\x88\xe4\x1f\xcd\xf1(\xa5\xba\xe5\x89\xe0\xfd\xab9+\x10U9\x9e\x1b\xca\xdc\xc4\xf1-C\x9a\x0f\xa9\xf1+\xc4\xf4\f\xcdw5\x06{\x9c=\xbe\xc9H\xe7\x14\x903\xad\xb4\xeb\x1eք\x99~\xb1\xb0\x95ƺ\x16\xe1\x05P\xa5\xe5k\xea\u05cd1\xd5'c\xce\x0e1\xbf\xfaѝcŝ~\nIMK\x02\xebH\xfc\x9d\xd8#\xc8-H\a\xa82]+>\xf0\"uA\xd3,\x80\xe8\x99\xe8\x8dI\xa2\xcd\xec\fVu\x99N\x90\x15K\xa7T\xb3\xa7c\xdd!\xbf\t\x99v:\x05\xe7\xb1՝J\x1c\x1ak\xfdl\xa8\x90A\xd4\xcd^+\xc5OY\xd6%\x88\x92ز$n\xdc\xfaܣ\x98\xea\xe6y\xfd$\xa4\v\x19\xc4\xfebu\x996\xcdtY\x15\xe80f\x15eZY\x99c\xe3>\x04\xfe\x8f\xe6hM5\x01[!\x8b\xda,\xd0ы9\xb34n\v\xea\xe9僱tDVL\xcc\xc4C\xf7\x05N\xf3\xbc\xfd\xa8\xcc2\x97\xf9\x9b\xc1\x97wM+#IJ\xf5\x9cw:\v\x93\xbd\u05few\x1a\x84W\xa8Ô{:\v\x951ysO\x9b\xf6枾\xb9\xa7o\xee頽\xb9\xa7o\xee\xe9\x9b{:\xde\xde\xdc\xd3N{sO\x93\xedG\n\x86+>\xb9=\xd1!\t\xab\xc4\x14\x8c9\xb4g\xe6\n\x99F7Em\x1d\x9a%\x19ҷ\xe3#G^\x03d\xbeˊ_\xe8NIM\x9b\xba\xd2\x1a\xbd&e\x9a\xb6d\xdcL\xfe\x1dU\x82\x17\xfe\x02\xd9\xf6\xa9\ttsis\xfd\xdc\xf1&]\xcd\xff\xdf\x04A\x9c\x8e\xd3\a\xee\xf9\xf7yݜ\xab~\xee\x1b\xc7\x01\x11\xe3?d^ybZ\xdbL2\xdb\xe9D\xfc)\v\x1fi9\xb8\\\xe8\x13\xd3\xf4\x12\xbf\xffشtX~\xad\xc2vH\u007f0v;2\xec\x19OƄ=\xa8lg\xb4ҵ\ra\"\xcd\xf0!\xf3o\xe8\xe2D\xf6\xf8\x8d\xd7\x14\x9b,\xbc\x87\x9d\xae'\x92\xbdg蚐\x827\x9dx\x17\xaerщ\xfd\xfbu\xff\x17\xa7C\x1a\xde\x04\xd6O\xd2\xed\xfcCF\x8a\xe7\xd5C7\xd7?n\xde\xf0\x98y(x\x13\x10\xb5\x01%\v/\x95\x11BO&\xe1k\xe5\xcf\r\xceV~\xf3\xd1kz\xb2\xde\xd2\x14\xbd&\xa9j\xde\xd4>#1o٫\x89\xd9$\xbc\x14\xa4!%\xf5n<\xa9n\x06ꒄ\xbbԃ\x89\x84\xe4\xba\xf4\x94\xba4\xf2\x00\x97 HM\xa4KveS\x93\xe6^'U.1A\xae\x93\xf66\v\xf2̴\xb8d\x82\xa5\xa5\xc0%'\xbeu\xd2\xd9\xe6\xa9u\"\xddm<\x89m\x16\xe4X\x92[J\xeaZ\x12\xae\xc9\tkM\x1a\xda\xfc\xf1\xea\xb3\xd2\xd4^>!\xfe%\x83\x9f\xd3IgI\xa9fI\x01\xd2<\xceI\xc9dKSȒ\xa8\xba4]\xacI\x05;1qR\x92\xd8q\x02ة\xa5̦\x86M\xa7}\x9d\x02;\x96\x10\x96\x90\xecu\x02d7\rl\xb1\x1b0+M3\x1d\xc6˄\xc46ok\x8b\xff\v\t|\ue8b5\xe9\xb9\xc0)\xa1\xda\xd7\xc1\x10\xe2}\xf4\xfa\xc6\xdc\xea\xe9\xc0\xd7;\xdbg\xb8\xd5\x13 o\xb7Pօ\x93Uѩ\xd3\xe1vxh\xea\x00\xfc\xae\xf9=\xea\xe6\xc0о~o\x04x\nd?@\x10\x16\x9e\xb0(\xe8\xbfGT\xc8|U\x9cL\xaf\x90l\xce\xf4\xdd@\xa8\u007f\x10J\xea\\\xf9\x04H~\xac\xcb\xf6\xac$H\xb1l\xc2\x191\xe9ig\xd7\xfb\xe8\xfc\xed/5\x9a\x03\xe8=\x9aƫ\x99\x14\xb3\xf6\x11Wؚ\x96\"\xbc\xa8J\x82N\xf2\xb5\x99\xfa\xaaez74\x1b\x1a>(of\x87\xb82,\xd2!mptJuR,4\x05B\xe9\x06\xc2\xc4\xf8\x14_zɫ\xa6\xd7\b\x95^\"XJr+^#`z\xad\x90iiд\xe4>7\xe9U\xd2k\x84NK\x82\xa7E\x1e`\xfa\xab\xa3\xd7zm\xf4\nA\xd4\xd9a\xd4\"ҥ\xbe&Z\x1cL%\xaco\xe6\xf5БǕ\x00r\xf2\xd5\xd0x@\x95\x00\xf1\xe8\xb5\xd0lH\x95\xb2\x0f\x86A׳\xdf\xfe$\xe76,\xba`K\xcdKH\xbb\xfb\x9a\u007fӓ\xf8\x96'\xf1f,\x05\xfb\xc47;\xcb\xdf\xea$\xd2\xf9\xcc`\xeb\xe4ԉor\x16\x85[g\x06\\'!\x9ez\x83s:\xe4:}\x9c6|{s\x86;\x91 a\xb3]\x9e}M\xa2M\x8ef\xf6\xc6i\x89h\xce\n\xe5 &\xea\xcf?\xb8k\x89%ϨW\xf76k\x8a;\xba)\r\x90\xc1\x9f\xa4\xca=oH\b;\xfeE\xefJ\xacu~\xa6o\\Z\x8f3\xd4\n\xf5wi\x16+A\x8a\x94\xc3\"\xbe\xf3\xb7k\xf8$\xb2]3\xc3\x04H\x9ew',l\xb5)\x85\x83\xcb\xe6\x92\U0009d7c0\xfe\xbe\\\x03\xfc\xa6\x9b\xdb\xe1N\xe9\x9f\t\xa8V\x96Uq\xa0\xe8\a.\xbb`\x9e'8\x93\xc2gC\r\xc4P\xe4-!\xfe\xbd\xeb\x8f\x18\xab\xc1\x19j\xddE\xd8'\xd8,\xd4\x01\xbe\xfd`/\x8a+bem\xe5\xb0\xe0#\xc5\bxPXl\x02\xe4T\x11\xcdEĚ\xbe)\xb7N\x1b\U000407f5\xafs\x9aB\xad\xfe\x88^\xa9۠\xa3b\xdeLx\b7\xb1\xb2X\xa1z\b\xb0M\xa7;*\xbdH\xd8N)\xaf\x99\xfd\xed\\\x91\xb0\xb8\xfb\xfb\xcf~AN\x96\xb8\xfeX\xfbk\xd0U%\x8cE\xa2t\\\xa8\x1f\xb4\x99\xb6o;\xfdĵ\v\xbb\xc5H;垑\xd3\xf78A\xe2\xac\xd5\xec{\xe5>#\xe9R\x84\xfd\xc7\xf8Ȏ:\xe90\xf1T\x9e\x83\xdeN\xc2\x12\xd6\xeaL\xb2\x06\xe2\xa3 Κ{\xbd\xbay\xa7,\xc9\teQ[\xfc\xfa\xa4\xd0|\x8f\x1b\xd5ު\xa9j\xa3=\x12\xfe\xf9h\xe0d\xa5Q\xa7Y\xef\r\xba\x8f\xd9;\x15\bd}e\xd6x\xa6%mS\x8f\xf7\x98t3\xfb\u007fz\xef\x8f\xfb\xac\xab\xf1\x12\xb8\xab\xa6*\xefE\x02e}\xe5ٔ\x82˾Dm&*W\x9b`V\xb3\xdap\x11A\x02\x82\xbe\xc6\xdey%\x97\xdb\x12\xf63\xbcl\x8bڷQ\xfel\t\xfd\x11\xfe5E\x93'\xab\b{\x9b\xeaKܯ\b\xfey\xec\x1c\xdd\a\\tq\xae\xc04\xf5i2\x9e\x03\xa1y`,\xd6x7\x85\xfax\n\xeb\n\xbe\xe0\xd3\xc8\xd7O\x8a\x16q|\x97\xe6\xf3T1糁\xb1r\xf3'\x97\xb8oFq\x92\xf0\x88\xb6諹A\xf7A\xf6\x11\x97\xeem\xba\xf8\x84\xe01\xb6\xfe\xa3\xdc\xfa\x83\x9b\x8c\xd6\xf4OG=&\x15\xd7I\xa55\xa5\xb0F\xb7\xd4\xd1G\x8bfϵr\xa3\x90\x04\x1b\xde\xfdRo\xdaڙ\xf0\u05ff]\xb4\xbbRd\x19V.d\xb9u\xffi\x8fK_\xfb6\xfe\xcb\x1d\xfcg\xa6\x95w\xb0\xed5\xfc\xe7\u007f_@0\xc0?\xe2?\xcfA\x1f\xff7\x00\x00\xff\xff\xfa\xbf\x85\x18\be\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VO\x8f\xeb4\x10\xbf\xe7S\x8c\x1e\x87w!\xe9{\xe2\x00\xca\r\x15\x0e+`\xb5\xda>\xed\x05qp\x9di;\xacc\x9b\xf1\xb8K\xf9\xf4\xc8v\xb2m\x93\x94]\x90\xf0-\xf6\xfc\xf9\xcdo\xfed\xaa\xba\xae+\xe5\xe9\t9\x90\xb3-(O\xf8\xa7\xa0M_\xa1y\xfe.4\xe4V\xc7\xcf\xd53ٮ\x85u\f\xe2\xfaG\f.\xb2\xc6\x1fpG\x96\x84\x9c\xadz\x14\xd5)Qm\x05\xa0\xacu\xa2\xd2uH\x9f\x00\xdaYag\fr\xbdG\xdb<\xc7-n#\x99\x0e9\x1b\x1f]\x1f?5\xdf6\x9f*\x00͘տP\x8fAT\xef[\xb0ј\n\xc0\xaa\x1e[\b\xc8II\x94\xc4\xc0\xf8G\xc4 \xa19\xa2Av\r\xb9*x\xd4\xc9\xf1\x9e]\xf4-\x9c\x1f\x8a\xfe\x00\xaa\x04\xb4ɦ6\xd9\xd4c1\x95_\r\x05\xf9\xe9\x96\xc4\xcf4Hy\x13Y\x99e@Y \x1c\x1c\xcb\xfd\xd9i\r!py!\xbb\x8fF\xf1\xa2r\x05\x10\xb4\xf3\xd8B\xd6\xf5JcW\x01\fLe[\xf5\xc0\xc5\xf1s1\xa7\x0fث\xe2\x04\xc0y\xb4\xdf?\xdc=}\xb3\xb9\xba\x06\xe80h&/\x99\xef\x85Ȁ\x02(\x18P\x808PZc\b\xa0#3Z\x81\x82\x12\xc8\xee\x1c\xf79G\xaf\xa6\x01\xd4\xd6E\x019 d\x05\xd9*\x03Ge\"~\r\xcavЫ\x130&/\x10텽,\x12\x1a\xf8\xc51f2[8\x88\xf8ЮV{\x92\xb1\xeb\xb4\xeb\xfbhIN\xab\xdc@\xb4\x8d\xe28\xac:<\xa2Y\x05\xda\u05ca\xf5\x81\x04\xb5Dƕ\xf2Tg\xe86w^\xd3w_\xf1Ч\xe1\xe3\x15V9\xa5\xca\n\xc2d\xf7\x17\x0f\xb9!\xfe!\x03\xa9\x1dJ}\x14\xd5\x12ř\xe8t\x95\xd8y\xfcq\xf3\x05F\xd79\x19S\xf63\xefg\xc5pNA\"\x8c\xec\x0e\xb9$qǮ\xcf6\xd1vޑ-ե\r\xa1\x9d\xd2\x1f\xe2\xb6'\tc\xed\xa6\\5\xb0Σ\b\xb6\b\xd1wJ\xb0k\xe0\xce\xc2Z\xf5h\xd6*\xe0\xff\x9e\x80\xc4t\xa8\x13\xb1\xefK\xc1\xe5\x14\x9d\n\x17\xd6.\x1e\xc61w#_\vݽ\xf1\xa8S\x06\x13\x89I\x9bv\xa4s{\xc0\xce1\xa8%\x95\xe6]H\xb2ƿ\xc42L\x92\x82f2_R\u007f\xbe\x8dfy\x9c䗃\n8\xbd\x9c`zH2S\xff\x86v\xa8O\xda`1Q\xa6\t\xbe\r%\x1d\xb4\xb1\x9f\xfb\xac\xe1\x1e_\x16n\x1fإɚ\xe7\xfa\xf5\xb9Q\x1bP\xfe7{\xb2\xb3p\xa7\x91\x15\xa9\xfc\x0f\xbb\x1c\xd5\x17\x03z0\x04\x1c\xadM};\x9b\x90\x19\xc8t\x92\xcfdH\xb0_@\xb3\x88\xe7\xce\xee\\\xde\x04Tr\xac\xa4\xf4\x13\x0e\xc9\x1e\xfc\x14\\\v\x06o纜\xf9\xf0z\x17\xa1\xe5\xe4?\xe9\u007fSN\xe3\x86\x18\x17}\xd7\x19\xd5\xe2C\xf2\xb8\xc4\xf8r\u007f\r(\xa31jk\xb0\x05\xe18\xd7.\xba\x8aY\x9d\xa6U3\x96\xday\x9fz\xa3\x80f\n\xa9O^\x0ehou\x03\xbc\xa8锿\xf2\f\xdb\xd3-\xd5\xf5\xebr8o\xa9R\xba-\xa4\xd9]\v-p\xf6.R\x16\xb3WJzq\xf3\x98\x11\xb2\xb9\x94\x1dg\xc6Uk\x8c\x8b\xc8<\x86\x9b\x10\x16\x93=\xbb\xcc滋\xf0\x828V\xfb1\xe0\xf3\xe8M\x9b\x9a\x17\xec\xee\xa7+\xee\x87\x0fW\xbbj\xfe\xd4\xcevT6t\xf8\xf5\xb7\xaaX\xc5\xeei\\0\xd3\xe5\xdf\x01\x00\x00\xff\xff\xfb\xb1p\x12\x1b\f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WO\x8f۶\x13\xbd\xfbS\f\x92\xc3^\"9\xc1\xef\xf0+t)\x82M\x0fA\xf3g\x11o\xf7R\xf4@\x93#\x8b]\x8aTgHm\xddO_\f)\xad\xbd\xb6\x9cl\x8aV\x17C\x149|\xf3\u07bc!\xbd\xaa\xaaj\xa5\x06{\x87\xc46\xf8\x06\xd4`\xf1ψ^\u07b8\xbe\xff\x81k\x1b\xd6\xe3\x9bս\xf5\xa6\x81\xeb\xc41\xf4_\x90C\"\x8dﰵ\xdeF\x1b\xfc\xaaǨ\x8c\x8a\xaaY\x01(\xefCT2\xcc\xf2\n\xa0\x83\x8f\x14\x9cC\xaav\xe8\xeb\xfb\xb4\xc5m\xb2\xce \xe5\xe0\xf3\xd6\xe3\xeb\xfa\xff\xf5\xeb\x15\x80&\xcc\xcbom\x8f\x1cU?4\xe0\x93s+\x00\xafzl`\f.\xf5\xc8^\r܅\xe8\x82.\x9b\xd5#:\xa4P۰\xe2\x01\xb5콣\x90\x86\x06\x0e\x1fJ\x88\tW\xc9\xe9.G\xdbL\xd1>L\xd1\xf2\x04g9\xfe\xfc\x95I\x1f,\xc7\xe8\xcdK\x9a,\xcaWO\xb0ƽT\x11G\xb2~w\xf4!\x1b\xe1+\n\x88\aJ!\x94\xa5%\x8b\x03\xd12$\xec|\xf9is\v\xf3\xd6Y\x8cS\xf63\uf1c5|\x90@\b\xb3\xbeE*\"\xb6\x14\xfa\x1c\x13\xbd\x19\x82\xf51\xbfhgџ\xd2\xcfi\xdb\xdb(\xba\xff\x91\x90\xa3hU\xc3u\xeeB\xb0EH\x83Q\x11M\r\xef=\\\xab\x1eݵb\xfc\xcf\x05\x10\xa6\xb9\x12b\x9f'\xc1q\x03=\x9d\\X;6\xd8\xd4\xde.\xe8\xb5\xec\xe4̀\xfa\x89\x81$\x8am\xed\xe4\xec6\xd0\t\xafj\xf6\xf9r\xbc\xfa\xc9\xf4e\x83C\xe9\xfe\xadݝ\x8e\x02(c\xf2١\xdc\xcdŵ_!l!\xef뼓\x14j\x1bH\x10\x8d\xd6 Us\x9e\x13\x92DS\xc2\x16\x9d\xe1\xfa,\xe4\x05\xces*\x84F4V\xee\x1c\xe8S$\x8f\x13\xf3᧬/\x94\x1f\x02\xe4ң~\xea\xb1>\xa27\xb9\xa9\x9f\xa1\t\xb9\x86\x19\r<\xd8\xd8\x15s\xb8\xe3C\xeay*\xc8s\x8f\xfb\xa5\xe1\x13\xec\xb7\x1d\xca\xcc\xd2N\x11\x185a\x14\x1c\x8cN\xcc+ά\x01>&\xce\xf6R\x8b\x11AZ\x845\xf3\xea{ܟ\x13\r\xdf\x12w:\xef\xbf\r\xf9J\xce\xc5\x190a\x8b\x84>.Z\\\xee\x1e\xe41bv\xb9\t\x9a\xc5\xe0\x1a\x87\xc8\xeb0\"\x8d\x16\x1f\xd6\x0f\x81\xee\xad\xdfUBxU\n\x81\xd7\xf9ް~\x99\u007f.\xa4|\xfb\xf9\xdd\xe7\x06\xde\x1a\x03!vH\xa2Z\x9b\xdc\\hG\xa7ݫ\xdcq_A\xb2\xe6ǫ\u007f\xc2K\x18\x8as\x9e\xc1\xcd&W\xff^N\xee\fJ(\xda\x14U\x02\x81\xf4M\x11\xbb\x9f\xd4,\xfda\xa9\x10gL\xdb\x10\x1c\xaa\xf3ғ\xeek\t\xcd9\xa4Jv\xf8\x1e\x9b\xcd\xce\xfd\x86\xc9n\xa6ibx\xc9j^6\x17B\xb9\x97\xe4[\x8a\xda\xe1%\xa3/p\xbc\x9cJ\xf5\xb8\xc1\xb3ZtT1\xf1\xf77\xe9\xbcl\x9a\xb9\x9d\x1a\xb5N$\x05=\xc5\\\xb8\xd0\xfc;\x8dz\xe8\x14/\xb8\xed\x19\xa8od\xe5,\x83\xb3-\xea\xbdvX\x02Bh\x17\xaa\xe9\xbb ˃>\xf5K\xa5\xf5vT֩\xadÅo\xbfxu\xf1\xebE\xf1\x17\xf5<\x1bd\xb9\xb5\x98\x06\"\xa5\x12{\xaa\xb2i䠾\xd2\xd2\\\xd0|:\xfd\xdb\xf1\xe2œ\u007f\x0e\xf9U\a_\xceDn\xe0\xd7\xdfV%*\x9a\xbb\xf9\xa2/\x83\u007f\a\x00\x00\xff\xff\xe4\xf3S\x85\xb2\r\x00\x00"), } diff --git a/pkg/apis/velero/v1/backup.go b/pkg/apis/velero/v1/backup.go index 4c43784f002..394fe969c9d 100644 --- a/pkg/apis/velero/v1/backup.go +++ b/pkg/apis/velero/v1/backup.go @@ -124,6 +124,11 @@ type BackupSpec struct { // The default value is 10 minute. // +optional CSISnapshotTimeout metav1.Duration `json:"csiSnapshotTimeout,omitempty"` + + // ItemOperationTimeout specifies the time used to wait for asynchronous BackupItemAction operations + // The default value is 1 hour. + // +optional + ItemOperationTimeout metav1.Duration `json:"itemOperationTimeout,omitempty"` } // BackupHooks contains custom behaviors that should be executed at different phases of the backup. @@ -221,7 +226,7 @@ const ( // BackupPhase is a string representation of the lifecycle phase // of a Velero backup. -// +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting +// +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;FinalizingAfterPluginOperations;FinalizingAfterPluginOperationsPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting type BackupPhase string const ( @@ -251,6 +256,23 @@ const ( // ongoing. The backup is not usable yet. BackupPhaseWaitingForPluginOperationsPartiallyFailed BackupPhase = "WaitingForPluginOperationsPartiallyFailed" + // BackupPhaseFinalizingAfterPluginOperations means the backup of + // Kubernetes resources, creation of snapshots, and other + // async plugin operations were successful and snapshot upload and + // other plugin operations are now complete, but the Backup is awaiting + // final update of resources modified during async operations. + // The backup is not usable yet. + BackupPhaseFinalizingAfterPluginOperations BackupPhase = "FinalizingAfterPluginOperations" + + // BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed means the backup of + // Kubernetes resources, creation of snapshots, and other + // async plugin operations were successful and snapshot upload and + // other plugin operations are now complete, but one or more errors + // occurred during backup or async operation processing, and the + // Backup is awaiting final update of resources modified during async + // operations. The backup is not usable yet. + BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed BackupPhase = "FinalizingAfterPluginOperationsPartiallyFailed" + // BackupPhaseCompleted means the backup has run successfully without // errors. BackupPhaseCompleted BackupPhase = "Completed" @@ -351,6 +373,21 @@ type BackupStatus struct { // completed CSI VolumeSnapshots for this backup. // +optional CSIVolumeSnapshotsCompleted int `json:"csiVolumeSnapshotsCompleted,omitempty"` + + // AsyncBackupItemOperationsAttempted is the total number of attempted + // async BackupItemAction operations for this backup. + // +optional + AsyncBackupItemOperationsAttempted int `json:"asyncBackupItemOperationsAttempted,omitempty"` + + // AsyncBackupItemOperationsCompleted is the total number of successfully completed + // async BackupItemAction operations for this backup. + // +optional + AsyncBackupItemOperationsCompleted int `json:"asyncBackupItemOperationsCompleted,omitempty"` + + // AsyncBackupItemOperationsFailed is the total number of async + // BackupItemAction operations for this backup which ended with an error. + // +optional + AsyncBackupItemOperationsFailed int `json:"asyncBackupItemOperationsFailed,omitempty"` } // BackupProgress stores information about the progress of a Backup's execution. diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 8c488dd4f6d..71ec427ed35 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -350,6 +350,7 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { } } out.CSISnapshotTimeout = in.CSISnapshotTimeout + out.ItemOperationTimeout = in.ItemOperationTimeout } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec. diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 24dacce5f50..699c610783c 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -45,6 +45,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" @@ -1140,17 +1141,18 @@ type recordResourcesAction struct { backups []velerov1.Backup additionalItems []velero.ResourceIdentifier operationID string + itemsToUpdate []velero.ResourceIdentifier } -func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { metadata, err := meta.Accessor(item) if err != nil { - return item, a.additionalItems, a.operationID, err + return item, a.additionalItems, a.operationID, a.itemsToUpdate, err } a.ids = append(a.ids, kubeutil.NamespaceAndName(metadata)) a.backups = append(a.backups, *backup) - return item, a.additionalItems, a.operationID, nil + return item, a.additionalItems, a.operationID, a.itemsToUpdate, nil } func (a *recordResourcesAction) AppliesTo() (velero.ResourceSelector, error) { @@ -1165,6 +1167,10 @@ func (a *recordResourcesAction) Cancel(operationID string, backup *velerov1.Back return nil } +func (a *recordResourcesAction) Name() string { + return "" +} + func (a *recordResourcesAction) ForResource(resource string) *recordResourcesAction { a.selector.IncludedResources = append(a.selector.IncludedResources, resource) return a @@ -1462,7 +1468,7 @@ func (a *appliesToErrorAction) AppliesTo() (velero.ResourceSelector, error) { return velero.ResourceSelector{}, errors.New("error calling AppliesTo") } -func (a *appliesToErrorAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (a *appliesToErrorAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { panic("not implemented") } @@ -1474,6 +1480,10 @@ func (a *appliesToErrorAction) Cancel(operationID string, backup *velerov1.Backu panic("not implemented") } +func (a *appliesToErrorAction) Name() string { + return "" +} + // TestBackupActionModifications runs backups with backup item actions that make modifications // to items in their Execute(...) methods and verifies that these modifications are // persisted to the backup tarball. Verification is done by inspecting the file contents @@ -1483,16 +1493,16 @@ func TestBackupActionModifications(t *testing.T) { // method modifies the item being passed in by calling the 'modify' function on it. modifyingActionGetter := func(modify func(*unstructured.Unstructured)) *pluggableAction { return &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { obj, ok := item.(*unstructured.Unstructured) if !ok { - return nil, nil, "", errors.Errorf("unexpected type %T", item) + return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) } res := obj.DeepCopy() modify(res) - return res, nil, "", nil + return res, nil, "", nil, nil }, } } @@ -1621,13 +1631,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, {GroupResource: kuberesource.Pods, Namespace: "ns-3", Name: "pod-3"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1652,13 +1662,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"}, {GroupResource: kuberesource.Pods, Namespace: "ns-3", Name: "pod-3"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1682,13 +1692,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1715,13 +1725,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1745,13 +1755,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1776,13 +1786,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { }, actions: []biav2.BackupItemAction{ &pluggableAction{ - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"}, {GroupResource: kuberesource.PersistentVolumes, Name: "pv-2"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -1807,13 +1817,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { actions: []biav2.BackupItemAction{ &pluggableAction{ selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}}, - executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { additionalItems := []velero.ResourceIdentifier{ {GroupResource: kuberesource.Pods, Namespace: "ns-4", Name: "pod-4"}, {GroupResource: kuberesource.Pods, Namespace: "ns-5", Name: "pod-5"}, } - return item, additionalItems, "", nil + return item, additionalItems, "", nil, nil }, }, }, @@ -2292,6 +2302,167 @@ func TestBackupWithSnapshots(t *testing.T) { } } +// TestBackupWithAsyncOperations runs backups which return operationIDs and +// verifies that the itemoperations are tracked as appropriate. Verification is done by +// looking at the backup request's itemOperationsList field. +func TestBackupWithAsyncOperations(t *testing.T) { + // completedOperationAction is a *pluggableAction, whose Execute(...) + // method returns an operationID which will always be done when calling Progress. + completedOperationAction := &pluggableAction{ + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + obj, ok := item.(*unstructured.Unstructured) + if !ok { + return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) + } + + return obj, nil, obj.GetName() + "-1", nil, nil + }, + progressFunc: func(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { + return velero.OperationProgress{ + Completed: true, + Description: "Done!", + }, nil + }, + } + + // incompleteOperationAction is a *pluggableAction, whose Execute(...) + // method returns an operationID which will never be done when calling Progress. + incompleteOperationAction := &pluggableAction{ + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + obj, ok := item.(*unstructured.Unstructured) + if !ok { + return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) + } + + return obj, nil, obj.GetName() + "-1", nil, nil + }, + progressFunc: func(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { + return velero.OperationProgress{ + Completed: false, + Description: "Working...", + }, nil + }, + } + + // noOperationAction is a *pluggableAction, whose Execute(...) + // method does not return an operationID. + noOperationAction := &pluggableAction{ + executeFunc: func(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { + obj, ok := item.(*unstructured.Unstructured) + if !ok { + return nil, nil, "", nil, errors.Errorf("unexpected type %T", item) + } + + return obj, nil, "", nil, nil + }, + } + + tests := []struct { + name string + req *Request + apiResources []*test.APIResource + actions []biav2.BackupItemAction + want []*itemoperation.BackupOperation + }{ + { + name: "action that starts a short-running process records operation", + req: &Request{ + Backup: defaultBackup().Result(), + }, + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-1").Result(), + ), + }, + actions: []biav2.BackupItemAction{ + completedOperationAction, + }, + want: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-1", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1"}, + OperationID: "pod-1-1", + }, + Status: itemoperation.OperationStatus{ + Phase: "InProgress", + }, + }, + }, + }, + { + name: "action that starts a long-running process records operation", + req: &Request{ + Backup: defaultBackup().Result(), + }, + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-2").Result(), + ), + }, + actions: []biav2.BackupItemAction{ + incompleteOperationAction, + }, + want: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-1", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-2"}, + OperationID: "pod-2-1", + }, + Status: itemoperation.OperationStatus{ + Phase: "InProgress", + }, + }, + }, + }, + { + name: "action that has no operation doesn't record one", + req: &Request{ + Backup: defaultBackup().Result(), + }, + apiResources: []*test.APIResource{ + test.Pods( + builder.ForPod("ns-1", "pod-3").Result(), + ), + }, + actions: []biav2.BackupItemAction{ + noOperationAction, + }, + want: []*itemoperation.BackupOperation{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var ( + h = newHarness(t) + backupFile = bytes.NewBuffer([]byte{}) + ) + + for _, resource := range tc.apiResources { + h.addItems(t, resource) + } + + err := h.backupper.Backup(h.log, tc.req, backupFile, tc.actions, nil) + assert.NoError(t, err) + + resultOper := *tc.req.GetItemOperationsList() + // set want Created times so it won't fail the assert.Equal test + for i, wantOper := range tc.want { + wantOper.Status.Created = resultOper[i].Status.Created + } + assert.Equal(t, tc.want, *tc.req.GetItemOperationsList()) + }) + } +} + // TestBackupWithInvalidHooks runs backups with invalid hook specifications and verifies // that an error is returned. func TestBackupWithInvalidHooks(t *testing.T) { @@ -2740,16 +2911,17 @@ func TestBackupWithPodVolume(t *testing.T) { } } -// pluggableAction is a backup item action that can be plugged with an Execute -// function body at runtime. +// pluggableAction is a backup item action that can be plugged with Execute +// and Progress function bodies at runtime. type pluggableAction struct { - selector velero.ResourceSelector - executeFunc func(runtime.Unstructured, *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) + selector velero.ResourceSelector + executeFunc func(runtime.Unstructured, *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) + progressFunc func(string, *velerov1.Backup) (velero.OperationProgress, error) } -func (a *pluggableAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (a *pluggableAction) Execute(item runtime.Unstructured, backup *velerov1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { if a.executeFunc == nil { - return item, nil, "", nil + return item, nil, "", nil, nil } return a.executeFunc(item, backup) @@ -2760,13 +2932,21 @@ func (a *pluggableAction) AppliesTo() (velero.ResourceSelector, error) { } func (a *pluggableAction) Progress(operationID string, backup *velerov1.Backup) (velero.OperationProgress, error) { - return velero.OperationProgress{}, nil + if a.progressFunc == nil { + return velero.OperationProgress{}, nil + } + + return a.progressFunc(operationID, backup) } func (a *pluggableAction) Cancel(operationID string, backup *velerov1.Backup) error { return nil } +func (a *pluggableAction) Name() string { + return "" +} + type harness struct { *test.APIServer backupper *kubernetesBackupper diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 54d68db1bc2..cd92899489a 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -42,7 +42,9 @@ import ( "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" vsv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/volumesnapshotter/v1" "github.com/vmware-tanzu/velero/pkg/podvolume" "github.com/vmware-tanzu/velero/pkg/util/boolptr" @@ -322,12 +324,36 @@ func (ib *itemBackupper) executeActions( } log.Info("Executing custom action") - // Note: we're ignoring the operationID returned from Execute for now, it will be used - // with the async plugin action implementation - updatedItem, additionalItemIdentifiers, _, err := action.Execute(obj, ib.backupRequest.Backup) + updatedItem, additionalItemIdentifiers, operationID, itemsToUpdate, err := action.Execute(obj, ib.backupRequest.Backup) if err != nil { return nil, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) } + + // If async plugin started async operation, add it to the ItemOperations list + if operationID != "" { + resourceIdentifier := velero.ResourceIdentifier{ + GroupResource: groupResource, + Namespace: namespace, + Name: name, + } + now := metav1.Now() + newOperation := itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + BackupName: ib.backupRequest.Backup.Name, + BackupUID: string(ib.backupRequest.Backup.UID), + BackupItemAction: action.Name(), + ResourceIdentifier: resourceIdentifier, + OperationID: operationID, + ItemsToUpdate: itemsToUpdate, + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &now, + }, + } + itemOperList := ib.backupRequest.GetItemOperationsList() + *itemOperList = append(*itemOperList, &newOperation) + } u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()} mustInclude := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation] == "true" diff --git a/pkg/backup/request.go b/pkg/backup/request.go index a94faa2d77c..24b7ecaa67f 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -24,6 +24,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/volume" @@ -51,6 +52,16 @@ type Request struct { PodVolumeBackups []*velerov1api.PodVolumeBackup BackedUpItems map[itemKey]struct{} CSISnapshots []snapshotv1api.VolumeSnapshot + itemOperationsList *[]*itemoperation.BackupOperation +} + +// GetItemOperationsList returns ItemOperationsList, initializing it if necessary +func (r *Request) GetItemOperationsList() *[]*itemoperation.BackupOperation { + if r.itemOperationsList == nil { + list := []*itemoperation.BackupOperation{} + r.itemOperationsList = &list + } + return r.itemOperationsList } // BackupResourceList returns the list of backed up resources grouped by the API diff --git a/pkg/builder/backup_builder.go b/pkg/builder/backup_builder.go index f18ce490923..a599816bad5 100644 --- a/pkg/builder/backup_builder.go +++ b/pkg/builder/backup_builder.go @@ -245,3 +245,9 @@ func (b *BackupBuilder) CSISnapshotTimeout(timeout time.Duration) *BackupBuilder b.object.Spec.CSISnapshotTimeout.Duration = timeout return b } + +// ItemOperationTimeout sets the Backup's ItemOperationTimeout +func (b *BackupBuilder) ItemOperationTimeout(timeout time.Duration) *BackupBuilder { + b.object.Spec.ItemOperationTimeout.Duration = timeout + return b +} diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index 52aeab3d527..ee16a08524e 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -99,6 +99,7 @@ type CreateOptions struct { FromSchedule string OrderedResources string CSISnapshotTimeout time.Duration + ItemOperationTimeout time.Duration client veleroclient.Interface } @@ -124,6 +125,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.VarP(&o.Selector, "selector", "l", "Only back up resources matching this label selector.") flags.StringVar(&o.OrderedResources, "ordered-resources", "", "Mapping Kinds to an ordered list of specific resources of that Kind. Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon. Example: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'. Optional.") flags.DurationVar(&o.CSISnapshotTimeout, "csi-snapshot-timeout", o.CSISnapshotTimeout, "How long to wait for CSI snapshot creation before timeout.") + flags.DurationVar(&o.ItemOperationTimeout, "item-operation-timeout", o.ItemOperationTimeout, "How long to wait for async plugin operations before timeout.") f := flags.VarPF(&o.SnapshotVolumes, "snapshot-volumes", "", "Take snapshots of PersistentVolumes as part of the backup. If the parameter is not set, it is treated as setting to 'true'.") // this allows the user to just specify "--snapshot-volumes" as shorthand for "--snapshot-volumes=true" // like a normal bool flag @@ -335,7 +337,8 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro TTL(o.TTL). StorageLocation(o.StorageLocation). VolumeSnapshotLocations(o.SnapshotLocations...). - CSISnapshotTimeout(o.CSISnapshotTimeout) + CSISnapshotTimeout(o.CSISnapshotTimeout). + ItemOperationTimeout(o.ItemOperationTimeout) if len(o.OrderedResources) > 0 { orders, err := ParseOrderedResources(o.OrderedResources) if err != nil { diff --git a/pkg/cmd/cli/backup/create_test.go b/pkg/cmd/cli/backup/create_test.go index 89416302565..5e7b0161b62 100644 --- a/pkg/cmd/cli/backup/create_test.go +++ b/pkg/cmd/cli/backup/create_test.go @@ -37,6 +37,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) { o.OrderedResources = "pods=p1,p2;persistentvolumeclaims=pvc1,pvc2" orders, err := ParseOrderedResources(o.OrderedResources) o.CSISnapshotTimeout = 20 * time.Minute + o.ItemOperationTimeout = 20 * time.Minute assert.NoError(t, err) backup, err := o.BuildBackup(testNamespace) @@ -49,6 +50,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) { IncludeClusterResources: o.IncludeClusterResources.Value, OrderedResources: orders, CSISnapshotTimeout: metav1.Duration{Duration: o.CSISnapshotTimeout}, + ItemOperationTimeout: metav1.Duration{Duration: o.ItemOperationTimeout}, }, backup.Spec) assert.Equal(t, map[string]string{ diff --git a/pkg/cmd/cli/backup/logs.go b/pkg/cmd/cli/backup/logs.go index 554fa17137a..95c2387a315 100644 --- a/pkg/cmd/cli/backup/logs.go +++ b/pkg/cmd/cli/backup/logs.go @@ -63,8 +63,8 @@ func NewLogsCommand(f client.Factory) *cobra.Command { } switch backup.Status.Phase { - case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed: - // terminal phases, do nothing. + case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: + // terminal and waiting for plugin operations phases, do nothing. default: cmd.Exit("Logs for backup %q are not available until it's finished processing. Please wait "+ "until the backup has a phase of Completed or Failed and try again.", backupName) diff --git a/pkg/cmd/cli/schedule/create.go b/pkg/cmd/cli/schedule/create.go index abd819d3bfa..4b8c29d3e1d 100644 --- a/pkg/cmd/cli/schedule/create.go +++ b/pkg/cmd/cli/schedule/create.go @@ -148,6 +148,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { DefaultVolumesToFsBackup: o.BackupOptions.DefaultVolumesToFsBackup.Value, OrderedResources: orders, CSISnapshotTimeout: metav1.Duration{Duration: o.BackupOptions.CSISnapshotTimeout}, + ItemOperationTimeout: metav1.Duration{Duration: o.BackupOptions.ItemOperationTimeout}, }, Schedule: o.Schedule, UseOwnerReferencesInBackup: &o.UseOwnerReferencesInBackup, diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index ea998ba106b..e6a642cf8b6 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -109,7 +109,8 @@ const ( // the default TTL for a backup defaultBackupTTL = 30 * 24 * time.Hour - defaultCSISnapshotTimeout = 10 * time.Minute + defaultCSISnapshotTimeout = 10 * time.Minute + defaultItemOperationTimeout = 60 * time.Minute // defaultCredentialsDirectory is the path on disk where credential // files will be written to @@ -121,6 +122,7 @@ type serverConfig struct { pluginDir, metricsAddress, defaultBackupLocation string backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout time.Duration defaultBackupTTL, storeValidationFrequency, defaultCSISnapshotTimeout time.Duration + defaultItemOperationTimeout time.Duration restoreResourcePriorities restore.Priorities defaultVolumeSnapshotLocations map[string]string restoreOnly bool @@ -132,6 +134,7 @@ type serverConfig struct { formatFlag *logging.FormatFlag repoMaintenanceFrequency time.Duration garbageCollectionFrequency time.Duration + itemOperationSyncFrequency time.Duration defaultVolumesToFsBackup bool uploaderType string } @@ -153,6 +156,7 @@ func NewCommand(f client.Factory) *cobra.Command { backupSyncPeriod: defaultBackupSyncPeriod, defaultBackupTTL: defaultBackupTTL, defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, + defaultItemOperationTimeout: defaultItemOperationTimeout, storeValidationFrequency: defaultStoreValidationFrequency, podVolumeOperationTimeout: defaultPodVolumeOperationTimeout, restoreResourcePriorities: defaultRestorePriorities, @@ -228,8 +232,10 @@ func NewCommand(f client.Factory) *cobra.Command { command.Flags().DurationVar(&config.defaultBackupTTL, "default-backup-ttl", config.defaultBackupTTL, "How long to wait by default before backups can be garbage collected.") command.Flags().DurationVar(&config.repoMaintenanceFrequency, "default-repo-maintain-frequency", config.repoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default.") command.Flags().DurationVar(&config.garbageCollectionFrequency, "garbage-collection-frequency", config.garbageCollectionFrequency, "How often garbage collection is run for expired backups.") + command.Flags().DurationVar(&config.itemOperationSyncFrequency, "item-operation-sync-frequency", config.itemOperationSyncFrequency, "How often to check status on async backup/restore operations after backup processing.") command.Flags().BoolVar(&config.defaultVolumesToFsBackup, "default-volumes-to-fs-backup", config.defaultVolumesToFsBackup, "Backup all volumes with pod volume file system backup by default.") command.Flags().StringVar(&config.uploaderType, "uploader-type", config.uploaderType, "Type of uploader to handle the transfer of data of pod volumes") + command.Flags().DurationVar(&config.defaultItemOperationTimeout, "default-item-operation-timeout", config.defaultItemOperationTimeout, "How long to wait on asynchronous BackupItemActions and RestoreItemActions to complete before timing out.") return command } @@ -656,6 +662,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.config.defaultVolumesToFsBackup, s.config.defaultBackupTTL, s.config.defaultCSISnapshotTimeout, + s.config.defaultItemOperationTimeout, s.sharedInformerFactory.Velero().V1().VolumeSnapshotLocations().Lister(), defaultVolumeSnapshotLocations, s.metrics, @@ -719,13 +726,14 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } // Note: all runtime type controllers that can be disabled are grouped separately, below: enabledRuntimeControllers := map[string]struct{}{ - controller.ServerStatusRequest: {}, - controller.DownloadRequest: {}, - controller.Schedule: {}, - controller.BackupRepo: {}, - controller.BackupDeletion: {}, - controller.GarbageCollection: {}, - controller.BackupSync: {}, + controller.ServerStatusRequest: {}, + controller.DownloadRequest: {}, + controller.Schedule: {}, + controller.BackupRepo: {}, + controller.BackupDeletion: {}, + controller.GarbageCollection: {}, + controller.BackupSync: {}, + controller.AsyncBackupOperations: {}, } if s.config.restoreOnly { @@ -735,6 +743,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string controller.Schedule, controller.GarbageCollection, controller.BackupDeletion, + controller.AsyncBackupOperations, ) } @@ -829,6 +838,22 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } } + var backupOpsMap *controller.BackupItemOperationsMap + if _, ok := enabledRuntimeControllers[controller.AsyncBackupOperations]; ok { + r, m := controller.NewAsyncBackupOperationsReconciler( + s.logger, + s.mgr.GetClient(), + s.config.itemOperationSyncFrequency, + newPluginManager, + backupStoreGetter, + s.metrics, + ) + if err := r.SetupWithManager(s.mgr); err != nil { + s.logger.Fatal(err, "unable to create controller", "controller", controller.AsyncBackupOperations) + } + backupOpsMap = m + } + if _, ok := enabledRuntimeControllers[controller.DownloadRequest]; ok { r := controller.NewDownloadRequestReconciler( s.mgr.GetClient(), @@ -836,6 +861,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string newPluginManager, backupStoreGetter, s.logger, + backupOpsMap, ) if err := r.SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", controller.DownloadRequest) diff --git a/pkg/cmd/server/server_test.go b/pkg/cmd/server/server_test.go index ddf9fb9d378..d25066d65ff 100644 --- a/pkg/cmd/server/server_test.go +++ b/pkg/cmd/server/server_test.go @@ -89,6 +89,7 @@ func TestRemoveControllers(t *testing.T) { { name: "Remove all disable controllers", disabledControllers: []string{ + controller.AsyncBackupOperations, controller.Backup, controller.BackupDeletion, controller.BackupSync, @@ -127,11 +128,12 @@ func TestRemoveControllers(t *testing.T) { } enabledRuntimeControllers := map[string]struct{}{ - controller.ServerStatusRequest: {}, - controller.Schedule: {}, - controller.BackupDeletion: {}, - controller.BackupRepo: {}, - controller.DownloadRequest: {}, + controller.ServerStatusRequest: {}, + controller.Schedule: {}, + controller.BackupDeletion: {}, + controller.BackupRepo: {}, + controller.DownloadRequest: {}, + controller.AsyncBackupOperations: {}, } totalNumOriginalControllers := len(enabledControllers) + len(enabledRuntimeControllers) diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 9be8ed59ee0..5b56da181dd 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -35,6 +35,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/features" clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" + "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/volume" ) @@ -66,6 +67,8 @@ func DescribeBackup( case velerov1api.BackupPhaseCompleted: phaseString = color.GreenString(phaseString) case velerov1api.BackupPhaseDeleting: + case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: + case velerov1api.BackupPhaseFinalizingAfterPluginOperations, velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed: case velerov1api.BackupPhaseInProgress: case velerov1api.BackupPhaseNew: } @@ -166,6 +169,7 @@ func DescribeBackupSpec(d *Describer, spec velerov1api.BackupSpec) { d.Println() d.Printf("CSISnapshotTimeout:\t%s\n", spec.CSISnapshotTimeout.Duration) + d.Printf("ItemOperationTimeout:\t%s\n", spec.ItemOperationTimeout.Duration) d.Println() if len(spec.Hooks.Resources) == 0 { @@ -284,6 +288,31 @@ func DescribeBackupStatus(ctx context.Context, kbClient kbclient.Client, d *Desc d.Println() } + if status.AsyncBackupItemOperationsAttempted > 0 { + if !details { + d.Printf("Async Backup Item Operations:\t%d of %d completed successfully, %d failed (specify --details for more information)\n", status.AsyncBackupItemOperationsCompleted, status.AsyncBackupItemOperationsAttempted, status.AsyncBackupItemOperationsFailed) + return + } + + buf := new(bytes.Buffer) + if err := downloadrequest.Stream(ctx, kbClient, backup.Namespace, backup.Name, velerov1api.DownloadTargetKindBackupItemOperations, buf, downloadRequestTimeout, insecureSkipTLSVerify, caCertPath); err != nil { + d.Printf("Async Backup Item Operations:\t\n", err) + return + } + + var operations []*itemoperation.BackupOperation + if err := json.NewDecoder(buf).Decode(&operations); err != nil { + d.Printf("Async Backup Item Operations:\t\n", err) + return + } + + d.Printf("Async Backup Item Operations:\n") + for _, operation := range operations { + describeAsyncBackupItemOperation(d, operation) + } + return + } + if details { describeBackupResourceList(ctx, kbClient, d, backup, insecureSkipTLSVerify, caCertPath) d.Println() @@ -365,6 +394,40 @@ func describeSnapshot(d *Describer, pvName, snapshotID, volumeType, volumeAZ str d.Printf("\t\tIOPS:\t%s\n", iopsString) } +func describeAsyncBackupItemOperation(d *Describer, operation *itemoperation.BackupOperation) { + d.Printf("\tOperation for %s %s/%s:\n", operation.Spec.ResourceIdentifier, operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) + d.Printf("\t\tBackup Item Action Plugin:\t%s\n", operation.Spec.BackupItemAction) + d.Printf("\t\tOperation ID:\t%s\n", operation.Spec.OperationID) + if len(operation.Spec.ItemsToUpdate) > 0 { + d.Printf("\t\tItems to Update:\n") + } + for _, item := range operation.Spec.ItemsToUpdate { + d.Printf("\t\t\t%s %s/%s\n", item, item.Namespace, item.Name) + } + d.Printf("\t\tPhase:\t%s\n", operation.Status.Phase) + if operation.Status.Error != "" { + d.Printf("\t\tOperation Error:\t%s\n", operation.Status.Error) + } + if operation.Status.NTotal > 0 || operation.Status.NCompleted > 0 { + d.Printf("\t\tProgress:\t%v of %v complete (%s)\n", + operation.Status.NCompleted, + operation.Status.NTotal, + operation.Status.OperationUnits) + } + if operation.Status.Description != "" { + d.Printf("\t\tProgress description:\t%s\n", operation.Status.Description) + } + if operation.Status.Created != nil { + d.Printf("\t\tCreated:\t%s\n", operation.Status.Created.String()) + } + if operation.Status.Started != nil { + d.Printf("\t\tStarted:\t%s\n", operation.Status.Started.String()) + } + if operation.Status.Updated != nil { + d.Printf("\t\tUpdated:\t%s\n", operation.Status.Updated.String()) + } +} + // DescribeDeleteBackupRequests describes delete backup requests in human-readable format. func DescribeDeleteBackupRequests(d *Describer, requests []velerov1api.DeleteBackupRequest) { d.Printf("Deletion Attempts") diff --git a/pkg/controller/async_backup_operations_controller.go b/pkg/controller/async_backup_operations_controller.go new file mode 100644 index 00000000000..f4533f21fdb --- /dev/null +++ b/pkg/controller/async_backup_operations_controller.go @@ -0,0 +1,514 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/clock" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/metrics" + "github.com/vmware-tanzu/velero/pkg/persistence" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/util/encode" + "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +const ( + defaultAsyncBackupOperationsFrequency = 2 * time.Minute +) + +type operationsForBackup struct { + operations []*itemoperation.BackupOperation + changesSinceUpdate bool + errsSinceUpdate []string +} + +func (o *operationsForBackup) anyItemsToUpdate() bool { + for _, op := range o.operations { + if len(op.Spec.ItemsToUpdate) > 0 { + return true + } + } + return false +} +func (in *operationsForBackup) DeepCopy() *operationsForBackup { + if in == nil { + return nil + } + out := new(operationsForBackup) + in.DeepCopyInto(out) + return out +} + +func (in *operationsForBackup) DeepCopyInto(out *operationsForBackup) { + *out = *in + if in.operations != nil { + in, out := &in.operations, &out.operations + *out = make([]*itemoperation.BackupOperation, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(itemoperation.BackupOperation) + (*in).DeepCopyInto(*out) + } + } + } + if in.errsSinceUpdate != nil { + in, out := &in.errsSinceUpdate, &out.errsSinceUpdate + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +func (o *operationsForBackup) uploadProgress(backupStore persistence.BackupStore, backupName string) error { + if len(o.operations) > 0 { + var backupItemOperations *bytes.Buffer + backupItemOperations, errs := encodeToJSONGzip(o.operations, "backup item operations list") + if errs != nil { + return errors.Wrap(errs[0], "error encoding item operations json") + } + err := backupStore.PutBackupItemOperations(backupName, backupItemOperations) + if err != nil { + return errors.Wrap(err, "error uploading item operations json") + } + } + o.changesSinceUpdate = false + o.errsSinceUpdate = nil + return nil +} + +type BackupItemOperationsMap struct { + operations map[string]*operationsForBackup + opsLock sync.Mutex +} + +// If backup has changes not yet uploaded, upload them now +func (m *BackupItemOperationsMap) UpdateForBackup(backupStore persistence.BackupStore, backupName string) error { + // lock operations map + m.opsLock.Lock() + defer m.opsLock.Unlock() + + operations, ok := m.operations[backupName] + // if operations for this backup aren't found, or if there are no changes + // or errors since last update, do nothing + if !ok || (!operations.changesSinceUpdate && len(operations.errsSinceUpdate) == 0) { + return nil + } + if err := operations.uploadProgress(backupStore, backupName); err != nil { + return err + } + return nil +} + +type asyncBackupOperationsReconciler struct { + client.Client + logger logrus.FieldLogger + clock clock.Clock + frequency time.Duration + itemOperationsMap *BackupItemOperationsMap + newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager + backupStoreGetter persistence.ObjectBackupStoreGetter + metrics *metrics.ServerMetrics +} + +func NewAsyncBackupOperationsReconciler( + logger logrus.FieldLogger, + client client.Client, + frequency time.Duration, + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, + backupStoreGetter persistence.ObjectBackupStoreGetter, + metrics *metrics.ServerMetrics, +) (*asyncBackupOperationsReconciler, *BackupItemOperationsMap) { + abor := &asyncBackupOperationsReconciler{ + Client: client, + logger: logger, + clock: clock.RealClock{}, + frequency: frequency, + itemOperationsMap: &BackupItemOperationsMap{operations: make(map[string]*operationsForBackup)}, + newPluginManager: newPluginManager, + backupStoreGetter: backupStoreGetter, + metrics: metrics, + } + if abor.frequency <= 0 { + abor.frequency = defaultAsyncBackupOperationsFrequency + } + return abor, abor.itemOperationsMap +} + +func (c *asyncBackupOperationsReconciler) SetupWithManager(mgr ctrl.Manager) error { + s := kube.NewPeriodicalEnqueueSource(c.logger, mgr.GetClient(), &velerov1api.BackupList{}, c.frequency, kube.PeriodicalEnqueueSourceOption{}) + return ctrl.NewControllerManagedBy(mgr). + For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + return false + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). + Watches(s, nil). + Complete(c) +} + +// +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get +// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get +func (c *asyncBackupOperationsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := c.logger.WithField("async backup operations for backup", req.String()) + // FIXME: make this log.Debug + log.Info("asyncBackupOperationsReconciler getting backup") + + original := &velerov1api.Backup{} + if err := c.Get(ctx, req.NamespacedName, original); err != nil { + if apierrors.IsNotFound(err) { + log.WithError(err).Error("backup not found") + return ctrl.Result{}, nil + } + return ctrl.Result{}, errors.Wrapf(err, "error getting backup %s", req.String()) + } + backup := original.DeepCopy() + log.Debugf("backup: %s", backup.Name) + + log = c.logger.WithFields( + logrus.Fields{ + "backup": req.String(), + }, + ) + + switch backup.Status.Phase { + case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed: + // only process backups waiting for plugin operations to complete + default: + log.Debug("Backup has no ongoing async plugin operations, skipping") + return ctrl.Result{}, nil + } + + loc := &velerov1api.BackupStorageLocation{} + if err := c.Get(ctx, client.ObjectKey{ + Namespace: req.Namespace, + Name: backup.Spec.StorageLocation, + }, loc); err != nil { + if apierrors.IsNotFound(err) { + log.Warnf("Cannot check progress on async Backup operations because backup storage location %s does not exist; marking backup PartiallyFailed", backup.Spec.StorageLocation) + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } else { + log.Warnf("Cannot check progress on async Backup operations because backup storage location %s could not be retrieved: %s; marking backup PartiallyFailed", backup.Spec.StorageLocation, err.Error()) + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } + err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &operationsForBackup{errsSinceUpdate: []string{err.Error()}}, false, false) + if err2 != nil { + log.WithError(err2).Error("error updating Backup") + } + return ctrl.Result{}, errors.Wrap(err, "error getting backup storage location") + } + + if loc.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly { + log.Infof("Cannot check progress on async Backup operations because backup storage location %s is currently in read-only mode; marking backup PartiallyFailed", loc.Name) + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + + err := c.updateBackupAndOperationsJSON(ctx, original, backup, nil, &operationsForBackup{errsSinceUpdate: []string{"BSL is read-only"}}, false, false) + if err != nil { + log.WithError(err).Error("error updating Backup") + } + return ctrl.Result{}, nil + } + + pluginManager := c.newPluginManager(c.logger) + defer pluginManager.CleanupClients() + backupStore, err := c.backupStoreGetter.Get(loc, pluginManager, c.logger) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "error getting backup store") + } + + operations, err := c.getOperationsForBackup(backupStore, backup.Name) + if err != nil { + err2 := c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, &operationsForBackup{errsSinceUpdate: []string{err.Error()}}, false, false) + if err2 != nil { + return ctrl.Result{}, errors.Wrap(err2, "error updating Backup") + } + return ctrl.Result{}, errors.Wrap(err, "error getting backup operations") + } + stillInProgress, changes, opsCompleted, opsFailed, errs := getBackupItemOperationProgress(backup, pluginManager, operations.operations) + // if len(errs)>0, need to update backup errors and error log + operations.errsSinceUpdate = append(operations.errsSinceUpdate, errs...) + backup.Status.Errors += len(operations.errsSinceUpdate) + asyncCompletionChanges := false + if backup.Status.AsyncBackupItemOperationsCompleted != opsCompleted || backup.Status.AsyncBackupItemOperationsFailed != opsFailed { + asyncCompletionChanges = true + backup.Status.AsyncBackupItemOperationsCompleted = opsCompleted + backup.Status.AsyncBackupItemOperationsFailed = opsFailed + } + if changes { + operations.changesSinceUpdate = true + } + + // if stillInProgress is false, backup is complete and needs update + // if operations.errsSinceUpdate is not empty, then backup phase needs to change to + // BackupPhaseWaitingForPluginOperationsPartiallyFailed and needs update + // If the only changes are incremental progress, then no write is necessary, progress can remain in memory + if !stillInProgress { + if len(operations.errsSinceUpdate) > 0 { + backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed + } + hasItemsForUpdate := operations.anyItemsToUpdate() + if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations && hasItemsForUpdate { + log.Infof("Marking backup %s FinalizingAfterPluginOperations", backup.Name) + backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperations + } else if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations { + log.Infof("Marking backup %s Completed", backup.Name) + backup.Status.Phase = velerov1api.BackupPhaseCompleted + } else if hasItemsForUpdate { + log.Infof("Marking backup %s FinalizingAfterPluginOperationsPartiallyFailed", backup.Name) + backup.Status.Phase = velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed + } else { + log.Infof("Marking backup %s PartiallyFailed", backup.Name) + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } + } + err = c.updateBackupAndOperationsJSON(ctx, original, backup, backupStore, operations, asyncCompletionChanges, changes) + if err != nil { + return ctrl.Result{}, errors.Wrap(err, "error updating Backup") + } + return ctrl.Result{}, nil +} + +func (c *asyncBackupOperationsReconciler) updateBackupAndOperationsJSON( + ctx context.Context, + original, backup *velerov1api.Backup, + backupStore persistence.BackupStore, + operations *operationsForBackup, + changes bool, + asyncCompletionChanges bool) error { + + backupScheduleName := backup.GetLabels()[velerov1api.ScheduleNameLabel] + + // update metrics and set completion time if complete + if backup.Status.Phase == velerov1api.BackupPhaseCompleted || + backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed { + + backup.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} + if backup.Status.StartTimestamp != nil { + backupDuration := backup.Status.CompletionTimestamp.Time.Sub(backup.Status.StartTimestamp.Time) + backupDurationSeconds := float64(backupDuration / time.Second) + c.metrics.RegisterBackupDuration(backupScheduleName, backupDurationSeconds) + } + if backup.Status.Phase == velerov1api.BackupPhaseCompleted { + c.metrics.RegisterBackupSuccess(backupScheduleName) + } + if backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed { + c.metrics.RegisterBackupPartialFailure(backupScheduleName) + } + } + if len(operations.errsSinceUpdate) > 0 { + c.metrics.RegisterBackupItemsErrorsGauge(backupScheduleName, backup.Status.Errors) + // FIXME: download/upload results once https://github.com/vmware-tanzu/velero/pull/5576 is merged + } + removeIfComplete := true + defer func() { + // remove local operations list if complete + c.itemOperationsMap.opsLock.Lock() + if removeIfComplete && (backup.Status.Phase == velerov1api.BackupPhaseCompleted || + backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed) { + + c.deleteOperationsForBackup(backup.Name) + } else if changes { + c.putOperationsForBackup(operations, backup.Name) + } + c.itemOperationsMap.opsLock.Unlock() + }() + + // update backup and upload progress if errs or complete + if len(operations.errsSinceUpdate) > 0 || + backup.Status.Phase == velerov1api.BackupPhaseCompleted || + backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed { + // update file store + if backupStore != nil { + backupJSON := new(bytes.Buffer) + if err := encode.EncodeTo(backup, "json", backupJSON); err != nil { + removeIfComplete = false + return errors.Wrap(err, "error encoding backup json") + } + err := backupStore.PutBackupMetadata(backup.Name, backupJSON) + if err != nil { + removeIfComplete = false + return errors.Wrap(err, "error uploading backup json") + } + if err := operations.uploadProgress(backupStore, backup.Name); err != nil { + removeIfComplete = false + return err + } + } + // update backup + err := c.Client.Patch(ctx, backup, client.MergeFrom(original)) + if err != nil { + removeIfComplete = false + return errors.Wrapf(err, "error updating Backup %s", backup.Name) + } + } else if asyncCompletionChanges { + // If backup is still incomplete and no new errors are found but there are some new operations + // completed, patch backup to reflect new completion numbers, but don't upload detailed json file + err := c.Client.Patch(ctx, backup, client.MergeFrom(original)) + if err != nil { + return errors.Wrapf(err, "error updating Backup %s", backup.Name) + } + } + return nil +} + +// returns a deep copy so we can minimize the time the map is locked +func (c *asyncBackupOperationsReconciler) getOperationsForBackup( + backupStore persistence.BackupStore, + backupName string) (*operationsForBackup, error) { + var err error + // lock operations map + c.itemOperationsMap.opsLock.Lock() + defer c.itemOperationsMap.opsLock.Unlock() + + operations, ok := c.itemOperationsMap.operations[backupName] + if !ok || len(operations.operations) == 0 { + operations = &operationsForBackup{} + operations.operations, err = backupStore.GetBackupItemOperations(backupName) + if err == nil { + c.itemOperationsMap.operations[backupName] = operations + } + } + return operations.DeepCopy(), err +} + +func (c *asyncBackupOperationsReconciler) putOperationsForBackup( + operations *operationsForBackup, + backupName string) { + if operations != nil { + c.itemOperationsMap.operations[backupName] = operations + } +} + +func (c *asyncBackupOperationsReconciler) deleteOperationsForBackup(backupName string) { + if _, ok := c.itemOperationsMap.operations[backupName]; ok { + delete(c.itemOperationsMap.operations, backupName) + } + return +} + +func getBackupItemOperationProgress( + backup *velerov1api.Backup, + pluginManager clientmgmt.Manager, + operationsList []*itemoperation.BackupOperation) (bool, bool, int, int, []string) { + inProgressOperations := false + changes := false + var errs []string + var completedCount, failedCount int + + for _, operation := range operationsList { + if operation.Status.Phase == itemoperation.OperationPhaseInProgress { + bia, err := pluginManager.GetBackupItemActionV2(operation.Spec.BackupItemAction) + if err != nil { + operation.Status.Phase = itemoperation.OperationPhaseFailed + operation.Status.Error = err.Error() + errs = append(errs, err.Error()) + changes = true + failedCount++ + continue + } + operationProgress, err := bia.Progress(operation.Spec.OperationID, backup) + if err != nil { + operation.Status.Phase = itemoperation.OperationPhaseFailed + operation.Status.Error = err.Error() + errs = append(errs, err.Error()) + changes = true + failedCount++ + continue + } + if operation.Status.NCompleted != operationProgress.NCompleted { + operation.Status.NCompleted = operationProgress.NCompleted + changes = true + } + if operation.Status.NTotal != operationProgress.NTotal { + operation.Status.NTotal = operationProgress.NTotal + changes = true + } + if operation.Status.OperationUnits != operationProgress.OperationUnits { + operation.Status.OperationUnits = operationProgress.OperationUnits + changes = true + } + started := metav1.NewTime(operationProgress.Started) + if operation.Status.Started == nil || *(operation.Status.Started) != started { + operation.Status.Started = &started + changes = true + } + updated := metav1.NewTime(operationProgress.Updated) + if operation.Status.Updated == nil || *(operation.Status.Updated) != updated { + operation.Status.Updated = &updated + changes = true + } + + if operationProgress.Completed { + if operationProgress.Err != "" { + operation.Status.Phase = itemoperation.OperationPhaseFailed + operation.Status.Error = operationProgress.Err + errs = append(errs, operationProgress.Err) + changes = true + failedCount++ + continue + } + operation.Status.Phase = itemoperation.OperationPhaseCompleted + changes = true + completedCount++ + continue + } + // cancel operation if past timeout period + if operation.Status.Created.Time.Add(backup.Spec.ItemOperationTimeout.Duration).Before(time.Now()) { + _ = bia.Cancel(operation.Spec.OperationID, backup) + operation.Status.Phase = itemoperation.OperationPhaseFailed + operation.Status.Error = "Asynchronous action timed out" + errs = append(errs, operation.Status.Error) + changes = true + failedCount++ + continue + } + // if we reach this point, the operation is still running + inProgressOperations = true + } else if operation.Status.Phase == itemoperation.OperationPhaseCompleted { + completedCount++ + } else if operation.Status.Phase == itemoperation.OperationPhaseFailed { + failedCount++ + } + } + return inProgressOperations, changes, completedCount, failedCount, errs +} diff --git a/pkg/controller/async_backup_operations_controller_test.go b/pkg/controller/async_backup_operations_controller_test.go new file mode 100644 index 00000000000..fd14bd6b8b4 --- /dev/null +++ b/pkg/controller/async_backup_operations_controller_test.go @@ -0,0 +1,308 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/clock" + ctrl "sigs.k8s.io/controller-runtime" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/metrics" + persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2mocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks/backupitemaction/v2" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +var ( + pluginManager = &pluginmocks.Manager{} + backupStore = &persistencemocks.BackupStore{} + bia = &biav2mocks.BackupItemAction{} +) + +func mockAsyncBackupOperationsReconciler(fakeClient kbclient.Client, fakeClock *clock.FakeClock, freq time.Duration) (*asyncBackupOperationsReconciler, *BackupItemOperationsMap) { + abor, biaMap := NewAsyncBackupOperationsReconciler( + logrus.StandardLogger(), + fakeClient, + freq, + func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewFakeSingleObjectBackupStoreGetter(backupStore), + metrics.NewServerMetrics(), + ) + abor.clock = fakeClock + return abor, biaMap +} + +func TestAsyncBackupOperationsReconcile(t *testing.T) { + fakeClock := clock.NewFakeClock(time.Now()) + metav1Now := metav1.NewTime(fakeClock.Now()) + + defaultBackupLocation := builder.ForBackupStorageLocation(velerov1api.DefaultNamespace, "default").Result() + + tests := []struct { + name string + backup *velerov1api.Backup + backupOperations []*itemoperation.BackupOperation + backupLocation *velerov1api.BackupStorageLocation + operationComplete bool + operationErr string + expectError bool + expectPhase velerov1api.BackupPhase + }{ + { + name: "WaitingForPluginOperations backup with completed operations is completed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + backupLocation: defaultBackupLocation, + operationComplete: true, + expectPhase: velerov1api.BackupPhaseCompleted, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-1", + BackupUID: "foo", + BackupItemAction: "foo", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-1", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperations backup with incomplete operations is still incomplete", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + backupLocation: defaultBackupLocation, + operationComplete: false, + expectPhase: velerov1api.BackupPhaseWaitingForPluginOperations, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-2", + BackupUID: "foo-2", + BackupItemAction: "foo-2", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-2", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperations backup with completed failed operations is PartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-3"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + backupLocation: defaultBackupLocation, + operationComplete: true, + operationErr: "failed", + expectPhase: velerov1api.BackupPhasePartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-3", + BackupUID: "foo-3", + BackupItemAction: "foo-3", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-3", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperationsPartiallyFailed backup with completed operations is PartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), + backupLocation: defaultBackupLocation, + operationComplete: true, + expectPhase: velerov1api.BackupPhasePartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-4", + BackupUID: "foo-4", + BackupItemAction: "foo-4", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-4", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperationsPartiallyFailed backup with incomplete operations is still incomplete", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-2"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), + backupLocation: defaultBackupLocation, + operationComplete: false, + expectPhase: velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-5", + BackupUID: "foo-5", + BackupItemAction: "foo-5", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-5", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + { + name: "WaitingForPluginOperationsPartiallyFailed backup with completed failed operations is PartiallyFailed", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-3"). + StorageLocation("default"). + ItemOperationTimeout(60 * time.Minute). + ObjectMeta(builder.WithUID("foo")). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), + backupLocation: defaultBackupLocation, + operationComplete: true, + operationErr: "failed", + expectPhase: velerov1api.BackupPhasePartiallyFailed, + backupOperations: []*itemoperation.BackupOperation{ + { + Spec: itemoperation.BackupOperationSpec{ + BackupName: "backup-6", + BackupUID: "foo-6", + BackupItemAction: "foo-6", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns-1", + Name: "pod-1", + }, + OperationID: "operation-6", + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseInProgress, + Created: &metav1Now, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.backup == nil { + return + } + + initObjs := []runtime.Object{} + initObjs = append(initObjs, test.backup) + + if test.backupLocation != nil { + initObjs = append(initObjs, test.backupLocation) + } + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) + reconciler, _ := mockAsyncBackupOperationsReconciler(fakeClient, fakeClock, defaultAsyncBackupOperationsFrequency) + pluginManager.On("CleanupClients").Return(nil) + backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil) + backupStore.On("PutBackupItemOperations", mock.Anything, mock.Anything).Return(nil) + backupStore.On("PutBackupMetadata", mock.Anything, mock.Anything).Return(nil) + for _, operation := range test.backupOperations { + bia.On("Progress", operation.Spec.OperationID, mock.Anything). + Return(velero.OperationProgress{ + Completed: test.operationComplete, + Err: test.operationErr, + }, nil) + pluginManager.On("GetBackupItemActionV2", operation.Spec.BackupItemAction).Return(bia, nil) + } + _, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}}) + gotErr := err != nil + assert.Equal(t, test.expectError, gotErr) + + backupAfter := velerov1api.Backup{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{ + Namespace: test.backup.Namespace, + Name: test.backup.Name, + }, &backupAfter) + + require.NoError(t, err) + assert.Equal(t, test.expectPhase, backupAfter.Status.Phase) + }) + } +} diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index be1c1872e77..7f3be76c678 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -76,27 +76,28 @@ import ( type backupController struct { *genericController - discoveryHelper discovery.Helper - backupper pkgbackup.Backupper - lister velerov1listers.BackupLister - client velerov1client.BackupsGetter - kbClient kbclient.Client - clock clock.Clock - backupLogLevel logrus.Level - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - backupTracker BackupTracker - defaultBackupLocation string - defaultVolumesToFsBackup bool - defaultBackupTTL time.Duration - defaultCSISnapshotTimeout time.Duration - snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister - defaultSnapshotLocations map[string]string - metrics *metrics.ServerMetrics - backupStoreGetter persistence.ObjectBackupStoreGetter - formatFlag logging.Format - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister - volumeSnapshotClient snapshotterClientSet.Interface - credentialFileStore credentials.FileStore + discoveryHelper discovery.Helper + backupper pkgbackup.Backupper + lister velerov1listers.BackupLister + client velerov1client.BackupsGetter + kbClient kbclient.Client + clock clock.Clock + backupLogLevel logrus.Level + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + backupTracker BackupTracker + defaultBackupLocation string + defaultVolumesToFsBackup bool + defaultBackupTTL time.Duration + defaultCSISnapshotTimeout time.Duration + defaultItemOperationTimeout time.Duration + snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister + defaultSnapshotLocations map[string]string + metrics *metrics.ServerMetrics + backupStoreGetter persistence.ObjectBackupStoreGetter + formatFlag logging.Format + volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister + volumeSnapshotClient snapshotterClientSet.Interface + credentialFileStore credentials.FileStore } func NewBackupController( @@ -113,6 +114,7 @@ func NewBackupController( defaultVolumesToFsBackup bool, defaultBackupTTL time.Duration, defaultCSISnapshotTimeout time.Duration, + defaultItemOperationTimeout time.Duration, volumeSnapshotLocationLister velerov1listers.VolumeSnapshotLocationLister, defaultSnapshotLocations map[string]string, metrics *metrics.ServerMetrics, @@ -123,28 +125,29 @@ func NewBackupController( credentialStore credentials.FileStore, ) Interface { c := &backupController{ - genericController: newGenericController(Backup, logger), - discoveryHelper: discoveryHelper, - backupper: backupper, - lister: backupInformer.Lister(), - client: client, - clock: &clock.RealClock{}, - backupLogLevel: backupLogLevel, - newPluginManager: newPluginManager, - backupTracker: backupTracker, - kbClient: kbClient, - defaultBackupLocation: defaultBackupLocation, - defaultVolumesToFsBackup: defaultVolumesToFsBackup, - defaultBackupTTL: defaultBackupTTL, - defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, - snapshotLocationLister: volumeSnapshotLocationLister, - defaultSnapshotLocations: defaultSnapshotLocations, - metrics: metrics, - backupStoreGetter: backupStoreGetter, - formatFlag: formatFlag, - volumeSnapshotLister: volumeSnapshotLister, - volumeSnapshotClient: volumeSnapshotClient, - credentialFileStore: credentialStore, + genericController: newGenericController(Backup, logger), + discoveryHelper: discoveryHelper, + backupper: backupper, + lister: backupInformer.Lister(), + client: client, + clock: &clock.RealClock{}, + backupLogLevel: backupLogLevel, + newPluginManager: newPluginManager, + backupTracker: backupTracker, + kbClient: kbClient, + defaultBackupLocation: defaultBackupLocation, + defaultVolumesToFsBackup: defaultVolumesToFsBackup, + defaultBackupTTL: defaultBackupTTL, + defaultCSISnapshotTimeout: defaultCSISnapshotTimeout, + defaultItemOperationTimeout: defaultItemOperationTimeout, + snapshotLocationLister: volumeSnapshotLocationLister, + defaultSnapshotLocations: defaultSnapshotLocations, + metrics: metrics, + backupStoreGetter: backupStoreGetter, + formatFlag: formatFlag, + volumeSnapshotLister: volumeSnapshotLister, + volumeSnapshotClient: volumeSnapshotClient, + credentialFileStore: credentialStore, } c.syncHandler = c.processBackup @@ -368,6 +371,11 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logg request.Spec.CSISnapshotTimeout.Duration = c.defaultCSISnapshotTimeout } + if request.Spec.ItemOperationTimeout.Duration == 0 { + // set default item operation timeout + request.Spec.ItemOperationTimeout.Duration = c.defaultItemOperationTimeout + } + // calculate expiration request.Status.Expiration = &metav1.Time{Time: c.clock.Now().Add(request.Spec.TTL.Duration)} @@ -706,10 +714,6 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { } } - // Mark completion timestamp before serializing and uploading. - // Otherwise, the JSON file in object storage has a CompletionTimestamp of 'null'. - backup.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} - backup.Status.VolumeSnapshotsAttempted = len(backup.VolumeSnapshots) for _, snap := range backup.VolumeSnapshots { if snap.Status.Phase == volume.SnapshotPhaseCompleted { @@ -724,11 +728,24 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { } } + // Iterate over backup item operations and update progress. + // Any errors on operations at this point should be added to backup errors. + // If any operations are still not complete, then back will not be set to + // Completed yet. + inProgressOperations, _, opsCompleted, opsFailed, errs := getBackupItemOperationProgress(backup.Backup, pluginManager, *backup.GetItemOperationsList()) + if len(errs) > 0 { + for err := range errs { + backupLog.Error(err) + } + } + + backup.Status.AsyncBackupItemOperationsAttempted = len(*backup.GetItemOperationsList()) + backup.Status.AsyncBackupItemOperationsCompleted = opsCompleted + backup.Status.AsyncBackupItemOperationsFailed = opsFailed + backup.Status.Warnings = logCounter.GetCount(logrus.WarnLevel) backup.Status.Errors = logCounter.GetCount(logrus.ErrorLevel) - recordBackupMetrics(backupLog, backup.Backup, backupFile, c.metrics) - if err := gzippedLogFile.Close(); err != nil { c.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)).WithError(err).Error("error closing gzippedLogFile") } @@ -741,10 +758,26 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { case len(fatalErrs) > 0: backup.Status.Phase = velerov1api.BackupPhaseFailed case logCounter.GetCount(logrus.ErrorLevel) > 0: - backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + if inProgressOperations { + backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed + } else { + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } default: - backup.Status.Phase = velerov1api.BackupPhaseCompleted + if inProgressOperations { + backup.Status.Phase = velerov1api.BackupPhaseWaitingForPluginOperations + } else { + backup.Status.Phase = velerov1api.BackupPhaseCompleted + } + } + // Mark completion timestamp before serializing and uploading. + // Otherwise, the JSON file in object storage has a CompletionTimestamp of 'null'. + if backup.Status.Phase == velerov1api.BackupPhaseFailed || + backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed || + backup.Status.Phase == velerov1api.BackupPhaseCompleted { + backup.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} } + recordBackupMetrics(backupLog, backup.Backup, backupFile, c.metrics) // re-instantiate the backup store because credentials could have changed since the original // instantiation, if this was a long-running backup @@ -776,9 +809,11 @@ func recordBackupMetrics(log logrus.FieldLogger, backup *velerov1api.Backup, bac } serverMetrics.SetBackupTarballSizeBytesGauge(backupScheduleName, backupSizeBytes) - backupDuration := backup.Status.CompletionTimestamp.Time.Sub(backup.Status.StartTimestamp.Time) - backupDurationSeconds := float64(backupDuration / time.Second) - serverMetrics.RegisterBackupDuration(backupScheduleName, backupDurationSeconds) + if backup.Status.CompletionTimestamp != nil { + backupDuration := backup.Status.CompletionTimestamp.Time.Sub(backup.Status.StartTimestamp.Time) + backupDurationSeconds := float64(backupDuration / time.Second) + serverMetrics.RegisterBackupDuration(backupScheduleName, backupDurationSeconds) + } serverMetrics.RegisterVolumeSnapshotAttempts(backupScheduleName, backup.Status.VolumeSnapshotsAttempted) serverMetrics.RegisterVolumeSnapshotSuccesses(backupScheduleName, backup.Status.VolumeSnapshotsCompleted) serverMetrics.RegisterVolumeSnapshotFailures(backupScheduleName, backup.Status.VolumeSnapshotsAttempted-backup.Status.VolumeSnapshotsCompleted) @@ -819,6 +854,12 @@ func persistBackup(backup *pkgbackup.Request, persistErrs = append(persistErrs, errs...) } + var backupItemOperations *bytes.Buffer + backupItemOperations, errs = encodeToJSONGzip(backup.GetItemOperationsList(), "backup item operations list") + if errs != nil { + persistErrs = append(persistErrs, errs...) + } + podVolumeBackups, errs := encodeToJSONGzip(backup.PodVolumeBackups, "pod volume backups list") if errs != nil { persistErrs = append(persistErrs, errs...) @@ -848,6 +889,7 @@ func persistBackup(backup *pkgbackup.Request, backupJSON = nil backupContents = nil nativeVolumeSnapshots = nil + backupItemOperations = nil backupResourceList = nil csiSnapshotJSON = nil csiSnapshotContentsJSON = nil @@ -861,6 +903,7 @@ func persistBackup(backup *pkgbackup.Request, Log: backupLog, PodVolumeBackups: podVolumeBackups, VolumeSnapshots: nativeVolumeSnapshots, + BackupItemOperations: backupItemOperations, BackupResourceList: backupResourceList, CSIVolumeSnapshots: csiSnapshotJSON, CSIVolumeSnapshotContents: csiSnapshotContentsJSON, diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index d29a586416a..e8114abcf86 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -148,6 +148,26 @@ func (b *backupSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) continue } + if backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperations || + backup.Status.Phase == velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed { + + if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { + log.Debugf("Skipping non-expired WaitingForPluginOperations backup %v", backup.Name) + continue + } + log.Debug("WaitingForPluginOperations Backup is past expiration, syncing for garbage collection") + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } + if backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperations || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed { + + if backup.Status.Expiration == nil || backup.Status.Expiration.After(time.Now()) { + log.Debugf("Skipping non-expired FinalizingAfterPluginOperations backup %v", backup.Name) + continue + } + log.Debug("FinalizingAfterPluginOperations Backup is past expiration, syncing for garbage collection") + backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed + } backup.Namespace = b.namespace backup.ResourceVersion = "" diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index 4f1e280c55f..2d3ac01f1e0 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -25,10 +25,12 @@ import ( . "github.com/onsi/gomega" "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" core "k8s.io/client-go/testing" @@ -155,9 +157,11 @@ func numBackups(c ctrlClient.WithWatch, ns string) (int, error) { var _ = Describe("Backup Sync Reconciler", func() { It("Test Backup Sync Reconciler basic function", func() { + fakeClock := clock.NewFakeClock(time.Now()) type cloudBackupData struct { - backup *velerov1api.Backup - podVolumeBackups []*velerov1api.PodVolumeBackup + backup *velerov1api.Backup + podVolumeBackups []*velerov1api.PodVolumeBackup + backupShouldSkipSync bool // backups waiting for plugin operations should not sync } tests := []struct { @@ -187,6 +191,98 @@ var _ = Describe("Backup Sync Reconciler", func() { }, }, }, + { + name: "backups waiting for plugin operations aren't synced", + namespace: "ns-1", + location: defaultLocation("ns-1"), + cloudBackups: []*cloudBackupData{ + { + backup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-2"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-3"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations).Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), + }, + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-4"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-5"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-6"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations).Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), + }, + backupShouldSkipSync: true, + }, + }, + }, + { + name: "expired backups waiting for plugin operations are synced", + namespace: "ns-1", + location: defaultLocation("ns-1"), + cloudBackups: []*cloudBackupData{ + { + backup: builder.ForBackup("ns-1", "backup-1"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-2"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-3"). + Phase(velerov1api.BackupPhaseWaitingForPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("ns-1", "pvb-1").Result(), + }, + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-4"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-5"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperationsPartiallyFailed). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + backupShouldSkipSync: true, + }, + { + backup: builder.ForBackup("ns-1", "backup-6"). + Phase(velerov1api.BackupPhaseFinalizingAfterPluginOperations). + Expiration(fakeClock.Now().Add(-time.Hour)).Result(), + podVolumeBackups: []*velerov1api.PodVolumeBackup{ + builder.ForPodVolumeBackup("ns-1", "pvb-2").Result(), + }, + backupShouldSkipSync: true, + }, + }, + }, { name: "all synced backups get created in Velero server's namespace", namespace: "velero", @@ -364,36 +460,42 @@ var _ = Describe("Backup Sync Reconciler", func() { Namespace: cloudBackupData.backup.Namespace, Name: cloudBackupData.backup.Name}, obj) - Expect(err).To(BeNil()) - - // did this cloud backup already exist in the cluster? - var existing *velerov1api.Backup - for _, obj := range test.existingBackups { - if obj.Name == cloudBackupData.backup.Name { - existing = obj - break + if cloudBackupData.backupShouldSkipSync && + (cloudBackupData.backup.Status.Expiration == nil || + cloudBackupData.backup.Status.Expiration.After(fakeClock.Now())) { + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } else { + Expect(err).To(BeNil()) + + // did this cloud backup already exist in the cluster? + var existing *velerov1api.Backup + for _, obj := range test.existingBackups { + if obj.Name == cloudBackupData.backup.Name { + existing = obj + break + } } - } - if existing != nil { - // if this cloud backup already exists in the cluster, make sure that what we get from the - // client is the existing backup, not the cloud one. + if existing != nil { + // if this cloud backup already exists in the cluster, make sure that what we get from the + // client is the existing backup, not the cloud one. - // verify that the in-cluster backup has its storage location populated, if it's not already. - expected := existing.DeepCopy() - expected.Spec.StorageLocation = test.location.Name + // verify that the in-cluster backup has its storage location populated, if it's not already. + expected := existing.DeepCopy() + expected.Spec.StorageLocation = test.location.Name - Expect(expected).To(BeEquivalentTo(obj)) - } else { - // verify that the storage location field and label are set properly - Expect(test.location.Name).To(BeEquivalentTo(obj.Spec.StorageLocation)) + Expect(expected).To(BeEquivalentTo(obj)) + } else { + // verify that the storage location field and label are set properly + Expect(test.location.Name).To(BeEquivalentTo(obj.Spec.StorageLocation)) - locationName := test.location.Name - if test.longLocationNameEnabled { - locationName = label.GetValidName(locationName) + locationName := test.location.Name + if test.longLocationNameEnabled { + locationName = label.GetValidName(locationName) + } + Expect(locationName).To(BeEquivalentTo(obj.Labels[velerov1api.StorageLocationLabel])) + Expect(len(obj.Labels[velerov1api.StorageLocationLabel]) <= validation.DNS1035LabelMaxLength).To(BeTrue()) } - Expect(locationName).To(BeEquivalentTo(obj.Labels[velerov1api.StorageLocationLabel])) - Expect(len(obj.Labels[velerov1api.StorageLocationLabel]) <= validation.DNS1035LabelMaxLength).To(BeTrue()) } // process the cloud pod volume backups for this backup, if any @@ -406,22 +508,28 @@ var _ = Describe("Backup Sync Reconciler", func() { Name: podVolumeBackup.Name, }, objPodVolumeBackup) - Expect(err).ShouldNot(HaveOccurred()) - - // did this cloud pod volume backup already exist in the cluster? - var existingPodVolumeBackup *velerov1api.PodVolumeBackup - for _, objPodVolumeBackup := range test.existingPodVolumeBackups { - if objPodVolumeBackup.Name == podVolumeBackup.Name { - existingPodVolumeBackup = objPodVolumeBackup - break + if cloudBackupData.backupShouldSkipSync && + (cloudBackupData.backup.Status.Expiration == nil || + cloudBackupData.backup.Status.Expiration.After(fakeClock.Now())) { + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } else { + Expect(err).ShouldNot(HaveOccurred()) + + // did this cloud pod volume backup already exist in the cluster? + var existingPodVolumeBackup *velerov1api.PodVolumeBackup + for _, objPodVolumeBackup := range test.existingPodVolumeBackups { + if objPodVolumeBackup.Name == podVolumeBackup.Name { + existingPodVolumeBackup = objPodVolumeBackup + break + } } - } - if existingPodVolumeBackup != nil { - // if this cloud pod volume backup already exists in the cluster, make sure that what we get from the - // client is the existing backup, not the cloud one. - expected := existingPodVolumeBackup.DeepCopy() - Expect(expected).To(BeEquivalentTo(objPodVolumeBackup)) + if existingPodVolumeBackup != nil { + // if this cloud pod volume backup already exists in the cluster, make sure that what we get from the + // client is the existing backup, not the cloud one. + expected := existingPodVolumeBackup.DeepCopy() + Expect(expected).To(BeEquivalentTo(objPodVolumeBackup)) + } } } } diff --git a/pkg/controller/constants.go b/pkg/controller/constants.go index 55d1ac76532..b9ad7bfb254 100644 --- a/pkg/controller/constants.go +++ b/pkg/controller/constants.go @@ -17,6 +17,7 @@ limitations under the License. package controller const ( + AsyncBackupOperations = "async-backup-operations" Backup = "backup" BackupDeletion = "backup-deletion" BackupStorageLocation = "backup-storage-location" @@ -33,6 +34,7 @@ const ( // DisableableControllers is a list of controllers that can be disabled var DisableableControllers = []string{ + AsyncBackupOperations, Backup, BackupDeletion, BackupSync, diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index c40d7aed3a8..ca8be0f2266 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -41,6 +41,9 @@ type downloadRequestReconciler struct { newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupStoreGetter persistence.ObjectBackupStoreGetter + // used to force update of async backup item operations before processing download request + backupItemOperationsMap *BackupItemOperationsMap + log logrus.FieldLogger } @@ -51,13 +54,15 @@ func NewDownloadRequestReconciler( newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupStoreGetter persistence.ObjectBackupStoreGetter, log logrus.FieldLogger, + backupItemOperationsMap *BackupItemOperationsMap, ) *downloadRequestReconciler { return &downloadRequestReconciler{ - client: client, - clock: clock, - newPluginManager: newPluginManager, - backupStoreGetter: backupStoreGetter, - log: log, + client: client, + clock: clock, + newPluginManager: newPluginManager, + backupStoreGetter: backupStoreGetter, + backupItemOperationsMap: backupItemOperationsMap, + log: log, } } @@ -157,6 +162,13 @@ func (r *downloadRequestReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, errors.WithStack(err) } + // If this is a request for backup item operations, force update of in-memory operations that + // are not yet uploaded + if downloadRequest.Spec.Target.Kind == velerov1api.DownloadTargetKindBackupItemOperations && + r.backupItemOperationsMap != nil { + // ignore errors here. If we can't upload anything here, process the download as usual + _ = r.backupItemOperationsMap.UpdateForBackup(backupStore, backupName) + } if downloadRequest.Status.DownloadURL, err = backupStore.GetDownloadURL(downloadRequest.Spec.Target); err != nil { return ctrl.Result{Requeue: true}, errors.WithStack(err) } diff --git a/pkg/controller/download_request_controller_test.go b/pkg/controller/download_request_controller_test.go index 340a6e5ce28..574722545bf 100644 --- a/pkg/controller/download_request_controller_test.go +++ b/pkg/controller/download_request_controller_test.go @@ -115,6 +115,7 @@ var _ = Describe("Download Request Reconciler", func() { func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, NewFakeObjectBackupStoreGetter(backupStores), velerotest.NewLogger(), + nil, ) if test.backupLocation != nil && test.expectGetsURL { diff --git a/pkg/itemoperation/backup_operation.go b/pkg/itemoperation/backup_operation.go index 94979ceeb9b..5c226f13e47 100644 --- a/pkg/itemoperation/backup_operation.go +++ b/pkg/itemoperation/backup_operation.go @@ -16,6 +16,10 @@ limitations under the License. package itemoperation +import ( + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + // BackupOperation stores information about an async item operation // started by a BackupItemAction plugin (v2 or later) type BackupOperation struct { @@ -24,6 +28,21 @@ type BackupOperation struct { Status OperationStatus `json:"status"` } +func (in *BackupOperation) DeepCopy() *BackupOperation { + if in == nil { + return nil + } + out := new(BackupOperation) + in.DeepCopyInto(out) + return out +} + +func (in *BackupOperation) DeepCopyInto(out *BackupOperation) { + *out = *in + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + type BackupOperationSpec struct { // BackupName is the name of the Velero backup this item operation // is associated with. @@ -37,8 +56,32 @@ type BackupOperationSpec struct { BackupItemAction string `json:"backupItemAction"` // Kubernetes resource identifier for the item - ResourceIdentifier string "json:resourceIdentifier" + ResourceIdentifier velero.ResourceIdentifier "json:resourceIdentifier" // OperationID returned by the BIA plugin OperationID string "json:operationID" + + // Items needing update after all async operations have completed + ItemsToUpdate []velero.ResourceIdentifier "json:itemsToUpdate" +} + +func (in *BackupOperationSpec) DeepCopy() *BackupOperationSpec { + if in == nil { + return nil + } + out := new(BackupOperationSpec) + in.DeepCopyInto(out) + return out +} + +func (in *BackupOperationSpec) DeepCopyInto(out *BackupOperationSpec) { + *out = *in + in.ResourceIdentifier.DeepCopyInto(&out.ResourceIdentifier) + if in.ItemsToUpdate != nil { + in, out := &in.ItemsToUpdate, &out.ItemsToUpdate + *out = make([]velero.ResourceIdentifier, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } diff --git a/pkg/itemoperation/restore_operation.go b/pkg/itemoperation/restore_operation.go index e3fe4d1f00d..50894983d1c 100644 --- a/pkg/itemoperation/restore_operation.go +++ b/pkg/itemoperation/restore_operation.go @@ -16,6 +16,10 @@ limitations under the License. package itemoperation +import ( + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + // RestoreOperation stores information about an async item operation // started by a RestoreItemAction plugin (v2 or later) type RestoreOperation struct { @@ -37,7 +41,7 @@ type RestoreOperationSpec struct { RestoreItemAction string `json:"restoreItemAction"` // Kubernetes resource identifier for the item - ResourceIdentifier string "json:resourceIdentifier" + ResourceIdentifier velero.ResourceIdentifier "json:resourceIdentifier" // OperationID returned by the RIA plugin OperationID string "json:operationID" diff --git a/pkg/itemoperation/shared.go b/pkg/itemoperation/shared.go index b9039b3ba22..b9c81ce0ba9 100644 --- a/pkg/itemoperation/shared.go +++ b/pkg/itemoperation/shared.go @@ -40,12 +40,19 @@ type OperationStatus struct { // Units that NCompleted,NTotal are measured in // i.e. "bytes" - OperationUnits int64 `json:"nTotal,omitempty"` + OperationUnits string `json:"operationUnits,omitempty"` + + // Description of progress made + // i.e. "processing", "Current phase: Running", etc. + Description string `json:"description,omitempty"` + + // Created records the time the item operation was created + Created *metav1.Time `json:"created,omitempty"` // Started records the time the item operation was started, if known // +optional // +nullable - Started *metav1.Time `json:"start,omitempty"` + Started *metav1.Time `json:"started,omitempty"` // Updated records the time the item operation was updated, if known. // +optional @@ -53,10 +60,35 @@ type OperationStatus struct { Updated *metav1.Time `json:"updated,omitempty"` } +func (in *OperationStatus) DeepCopy() *OperationStatus { + if in == nil { + return nil + } + out := new(OperationStatus) + in.DeepCopyInto(out) + return out +} + +func (in *OperationStatus) DeepCopyInto(out *OperationStatus) { + *out = *in + if in.Created != nil { + in, out := &in.Created, &out.Created + *out = (*in).DeepCopy() + } + if in.Started != nil { + in, out := &in.Started, &out.Started + *out = (*in).DeepCopy() + } + if in.Updated != nil { + in, out := &in.Updated, &out.Updated + *out = (*in).DeepCopy() + } +} + const ( // OperationPhaseNew means the item operation has been created and started // by the plugin - OperationPhaseInProgress OperationPhase = "New" + OperationPhaseInProgress OperationPhase = "InProgress" // OperationPhaseCompleted means the item operation was successfully completed // and can be used for restore. diff --git a/pkg/persistence/mocks/backup_store.go b/pkg/persistence/mocks/backup_store.go index 2caae0c83b8..a0aa047a0e9 100644 --- a/pkg/persistence/mocks/backup_store.go +++ b/pkg/persistence/mocks/backup_store.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +// Code generated by mockery v2.16.0. DO NOT EDIT. package mocks @@ -20,13 +21,15 @@ import ( io "io" mock "github.com/stretchr/testify/mock" + itemoperation "github.com/vmware-tanzu/velero/pkg/itemoperation" - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + persistence "github.com/vmware-tanzu/velero/pkg/persistence" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/itemoperation" - persistence "github.com/vmware-tanzu/velero/pkg/persistence" + volume "github.com/vmware-tanzu/velero/pkg/volume" + + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" ) // BackupStore is an autogenerated mock type for the BackupStore type @@ -106,6 +109,29 @@ func (_m *BackupStore) GetBackupContents(name string) (io.ReadCloser, error) { return r0, r1 } +// GetBackupItemOperations provides a mock function with given fields: name +func (_m *BackupStore) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) { + ret := _m.Called(name) + + var r0 []*itemoperation.BackupOperation + if rf, ok := ret.Get(0).(func(string) []*itemoperation.BackupOperation); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*itemoperation.BackupOperation) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetBackupMetadata provides a mock function with given fields: name func (_m *BackupStore) GetBackupMetadata(name string) (*v1.Backup, error) { ret := _m.Called(name) @@ -152,6 +178,75 @@ func (_m *BackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot return r0, r1 } +// GetCSIVolumeSnapshotClasses provides a mock function with given fields: name +func (_m *BackupStore) GetCSIVolumeSnapshotClasses(name string) ([]*volumesnapshotv1.VolumeSnapshotClass, error) { + ret := _m.Called(name) + + var r0 []*volumesnapshotv1.VolumeSnapshotClass + if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshotClass); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshotClass) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetCSIVolumeSnapshotContents provides a mock function with given fields: name +func (_m *BackupStore) GetCSIVolumeSnapshotContents(name string) ([]*volumesnapshotv1.VolumeSnapshotContent, error) { + ret := _m.Called(name) + + var r0 []*volumesnapshotv1.VolumeSnapshotContent + if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshotContent); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshotContent) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetCSIVolumeSnapshots provides a mock function with given fields: name +func (_m *BackupStore) GetCSIVolumeSnapshots(name string) ([]*volumesnapshotv1.VolumeSnapshot, error) { + ret := _m.Called(name) + + var r0 []*volumesnapshotv1.VolumeSnapshot + if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshot); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*volumesnapshotv1.VolumeSnapshot) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetDownloadURL provides a mock function with given fields: target func (_m *BackupStore) GetDownloadURL(target v1.DownloadTarget) (string, error) { ret := _m.Called(target) @@ -196,6 +291,29 @@ func (_m *BackupStore) GetPodVolumeBackups(name string) ([]*v1.PodVolumeBackup, return r0, r1 } +// GetRestoreItemOperations provides a mock function with given fields: name +func (_m *BackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.RestoreOperation, error) { + ret := _m.Called(name) + + var r0 []*itemoperation.RestoreOperation + if rf, ok := ret.Get(0).(func(string) []*itemoperation.RestoreOperation); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*itemoperation.RestoreOperation) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // IsValid provides a mock function with given fields: func (_m *BackupStore) IsValid() error { ret := _m.Called() @@ -247,13 +365,13 @@ func (_m *BackupStore) PutBackup(info persistence.BackupInfo) error { return r0 } -// PutRestoreLog provides a mock function with given fields: backup, restore, log -func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error { - ret := _m.Called(backup, restore, log) +// PutBackupItemOperations provides a mock function with given fields: backup, backupItemOperations +func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperations io.Reader) error { + ret := _m.Called(backup, backupItemOperations) var r0 error - if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { - r0 = rf(backup, restore, log) + if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { + r0 = rf(backup, backupItemOperations) } else { r0 = ret.Error(0) } @@ -261,13 +379,13 @@ func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reade return r0 } -// PutRestoreResults provides a mock function with given fields: backup, restore, results -func (_m *BackupStore) PutRestoreResults(backup string, restore string, results io.Reader) error { - ret := _m.Called(backup, restore, results) +// PutBackupMetadata provides a mock function with given fields: backup, backupMetadata +func (_m *BackupStore) PutBackupMetadata(backup string, backupMetadata io.Reader) error { + ret := _m.Called(backup, backupMetadata) var r0 error - if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { - r0 = rf(backup, restore, results) + if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { + r0 = rf(backup, backupMetadata) } else { r0 = ret.Error(0) } @@ -289,13 +407,13 @@ func (_m *BackupStore) PutRestoreItemOperations(backup string, restore string, r return r0 } -// PutBackupItemOperations provides a mock function with given fields: backup, backupItemOperations -func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperations io.Reader) error { - ret := _m.Called(backup, backupItemOperations) +// PutRestoreLog provides a mock function with given fields: backup, restore, log +func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error { + ret := _m.Called(backup, restore, log) var r0 error - if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { - r0 = rf(backup, backupItemOperations) + if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { + r0 = rf(backup, restore, log) } else { r0 = ret.Error(0) } @@ -303,27 +421,31 @@ func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperatio return r0 } -func (_m *BackupStore) GetCSIVolumeSnapshots(backup string) ([]*snapshotv1api.VolumeSnapshot, error) { - panic("Not implemented") - return nil, nil -} +// PutRestoreResults provides a mock function with given fields: backup, restore, results +func (_m *BackupStore) PutRestoreResults(backup string, restore string, results io.Reader) error { + ret := _m.Called(backup, restore, results) -func (_m *BackupStore) GetCSIVolumeSnapshotContents(backup string) ([]*snapshotv1api.VolumeSnapshotContent, error) { - panic("Not implemented") - return nil, nil -} + var r0 error + if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { + r0 = rf(backup, restore, results) + } else { + r0 = ret.Error(0) + } -func (_m *BackupStore) GetCSIVolumeSnapshotClasses(backup string) ([]*snapshotv1api.VolumeSnapshotClass, error) { - panic("Not implemented") - return nil, nil + return r0 } -func (_m *BackupStore) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) { - panic("implement me") - return nil, nil +type mockConstructorTestingTNewBackupStore interface { + mock.TestingT + Cleanup(func()) } -func (_m *BackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.RestoreOperation, error) { - panic("implement me") - return nil, nil +// NewBackupStore creates a new instance of BackupStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewBackupStore(t mockConstructorTestingTNewBackupStore) *BackupStore { + mock := &BackupStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock } diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 6609a0c6861..4a187d9b3e6 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -60,6 +60,7 @@ type BackupStore interface { ListBackups() ([]string, error) PutBackup(info BackupInfo) error + PutBackupMetadata(backup string, backupMetadata io.Reader) error PutBackupItemOperations(backup string, backupItemOperations io.Reader) error GetBackupMetadata(name string) (*velerov1api.Backup, error) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) @@ -312,6 +313,10 @@ func (s *objectBackupStore) GetBackupMetadata(name string) (*velerov1api.Backup, return backupObj, nil } +func (s *objectBackupStore) PutBackupMetadata(backup string, backupMetadata io.Reader) error { + return seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupMetadataKey(backup), backupMetadata) +} + func (s *objectBackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) { // if the volumesnapshots file doesn't exist, we don't want to return an error, since // a legacy backup or a backup with no snapshots would not have this file, so check for diff --git a/pkg/persistence/object_store_test.go b/pkg/persistence/object_store_test.go index 4636bc84601..b36735e8035 100644 --- a/pkg/persistence/object_store_test.go +++ b/pkg/persistence/object_store_test.go @@ -37,6 +37,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/plugin/velero" providermocks "github.com/vmware-tanzu/velero/pkg/plugin/velero/mocks" velerotest "github.com/vmware-tanzu/velero/pkg/test" @@ -461,14 +462,22 @@ func TestGetBackupItemOperations(t *testing.T) { operations := []*itemoperation.BackupOperation{ { Spec: itemoperation.BackupOperationSpec{ - BackupName: "test-backup", - ResourceIdentifier: "item-1", + BackupName: "test-backup", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns", + Name: "item-1", + }, }, }, { Spec: itemoperation.BackupOperationSpec{ - BackupName: "test-backup", - ResourceIdentifier: "item-2", + BackupName: "test-backup", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: kuberesource.Pods, + Namespace: "ns", + Name: "item-2", + }, }, }, } diff --git a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go index b3739796f75..c3121b1aac7 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action.go @@ -96,6 +96,11 @@ func (r *RestartableBackupItemAction) getDelegate() (biav2.BackupItemAction, err return r.getBackupItemAction() } +// Name returns the plugin's name. +func (r *RestartableBackupItemAction) Name() string { + return r.Key.Name +} + // AppliesTo restarts the plugin's process if needed, then delegates the call. func (r *RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { delegate, err := r.getDelegate() @@ -107,10 +112,10 @@ func (r *RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, erro } // Execute restarts the plugin's process if needed, then delegates the call. -func (r *RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (r *RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { delegate, err := r.getDelegate() if err != nil { - return nil, nil, "", err + return nil, nil, "", nil, err } return delegate.Execute(item, backup) @@ -148,15 +153,20 @@ func NewAdaptedV1RestartableBackupItemAction(v1Restartable *biav1cli.Restartable return r } +// Name restarts the plugin's name. +func (r *AdaptedV1RestartableBackupItemAction) Name() string { + return r.V1Restartable.Key.Name +} + // AppliesTo delegates to the v1 AppliesTo call. func (r *AdaptedV1RestartableBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { return r.V1Restartable.AppliesTo() } // Execute delegates to the v1 Execute call, returning an empty operationID. -func (r *AdaptedV1RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (r *AdaptedV1RestartableBackupItemAction) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { updatedItem, additionalItems, err := r.V1Restartable.Execute(item, backup) - return updatedItem, additionalItems, "", err + return updatedItem, additionalItems, "", nil, err } // Progress returns with an error since v1 plugins will never return an operationID, which means that diff --git a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go index 24e3c12a602..9d2f66c34b3 100644 --- a/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go +++ b/pkg/plugin/clientmgmt/backupitemaction/v2/restartable_backup_item_action_test.go @@ -145,8 +145,8 @@ func TestRestartableBackupItemActionDelegatedFunctions(t *testing.T) { restartabletest.RestartableDelegateTest{ Function: "Execute", Inputs: []interface{}{pv, b}, - ExpectedErrorOutputs: []interface{}{nil, ([]velero.ResourceIdentifier)(nil), "", errors.Errorf("reset error")}, - ExpectedDelegateOutputs: []interface{}{pvToReturn, additionalItems, "", errors.Errorf("delegate error")}, + ExpectedErrorOutputs: []interface{}{nil, ([]velero.ResourceIdentifier)(nil), "", ([]velero.ResourceIdentifier)(nil), errors.Errorf("reset error")}, + ExpectedDelegateOutputs: []interface{}{pvToReturn, additionalItems, "", ([]velero.ResourceIdentifier)(nil), errors.Errorf("delegate error")}, }, restartabletest.RestartableDelegateTest{ Function: "Progress", diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go index 925cf037c7b..b02c35f3c4b 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_client.go @@ -76,15 +76,15 @@ func (c *BackupItemActionGRPCClient) AppliesTo() (velero.ResourceSelector, error }, nil } -func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { itemJSON, err := json.Marshal(item.UnstructuredContent()) if err != nil { - return nil, nil, "", errors.WithStack(err) + return nil, nil, "", nil, errors.WithStack(err) } backupJSON, err := json.Marshal(backup) if err != nil { - return nil, nil, "", errors.WithStack(err) + return nil, nil, "", nil, errors.WithStack(err) } req := &protobiav2.ExecuteRequest{ @@ -95,12 +95,12 @@ func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup * res, err := c.grpcClient.Execute(context.Background(), req) if err != nil { - return nil, nil, "", common.FromGRPCError(err) + return nil, nil, "", nil, common.FromGRPCError(err) } var updatedItem unstructured.Unstructured if err := json.Unmarshal(res.Item, &updatedItem); err != nil { - return nil, nil, "", errors.WithStack(err) + return nil, nil, "", nil, errors.WithStack(err) } var additionalItems []velero.ResourceIdentifier @@ -118,7 +118,22 @@ func (c *BackupItemActionGRPCClient) Execute(item runtime.Unstructured, backup * additionalItems = append(additionalItems, newItem) } - return &updatedItem, additionalItems, res.OperationID, nil + var itemsToUpdate []velero.ResourceIdentifier + + for _, itm := range res.ItemsToUpdate { + newItem := velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: itm.Group, + Resource: itm.Resource, + }, + Namespace: itm.Namespace, + Name: itm.Name, + } + + itemsToUpdate = append(itemsToUpdate, newItem) + } + + return &updatedItem, additionalItems, res.OperationID, itemsToUpdate, nil } func (c *BackupItemActionGRPCClient) Progress(operationID string, backup *api.Backup) (velero.OperationProgress, error) { @@ -167,3 +182,9 @@ func (c *BackupItemActionGRPCClient) Cancel(operationID string, backup *api.Back return nil } + +// This shouldn't be called on the GRPC client since the RestartableBackupItemAction won't delegate +// this method +func (c *BackupItemActionGRPCClient) Name() string { + return "" +} diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go index 2de33ed7c46..246214e0ac4 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_server.go @@ -107,7 +107,7 @@ func (s *BackupItemActionGRPCServer) Execute( return nil, common.NewGRPCError(errors.WithStack(err)) } - updatedItem, additionalItems, operationID, err := impl.Execute(&item, &backup) + updatedItem, additionalItems, operationID, itemsToUpdate, err := impl.Execute(&item, &backup) if err != nil { return nil, common.NewGRPCError(err) } @@ -132,6 +132,9 @@ func (s *BackupItemActionGRPCServer) Execute( for _, item := range additionalItems { res.AdditionalItems = append(res.AdditionalItems, backupResourceIdentifierToProto(item)) } + for _, item := range itemsToUpdate { + res.ItemsToUpdate = append(res.ItemsToUpdate, backupResourceIdentifierToProto(item)) + } return res, nil } @@ -210,3 +213,9 @@ func backupResourceIdentifierToProto(id velero.ResourceIdentifier) *proto.Resour Name: id.Name, } } + +// This shouldn't be called on the GRPC server since the server won't ever receive this request, as +// the RestartableBackupItemAction in Velero won't delegate this to the server +func (c *BackupItemActionGRPCServer) Name() string { + return "" +} diff --git a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go index d9e2d2cea1d..a0c91d4e167 100644 --- a/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go +++ b/pkg/plugin/framework/backupitemaction/v2/backup_item_action_test.go @@ -97,6 +97,7 @@ func TestBackupItemActionGRPCServerExecute(t *testing.T) { implUpdatedItem runtime.Unstructured implAdditionalItems []velero.ResourceIdentifier implOperationID string + implItemsToUpdate []velero.ResourceIdentifier implError error expectError bool skipMock bool @@ -153,7 +154,7 @@ func TestBackupItemActionGRPCServerExecute(t *testing.T) { defer itemAction.AssertExpectations(t) if !test.skipMock { - itemAction.On("Execute", &validItemObject, &validBackupObject).Return(test.implUpdatedItem, test.implAdditionalItems, test.implOperationID, test.implError) + itemAction.On("Execute", &validItemObject, &validBackupObject).Return(test.implUpdatedItem, test.implAdditionalItems, test.implOperationID, test.implItemsToUpdate, test.implError) } s := &BackupItemActionGRPCServer{mux: &common.ServerMux{ diff --git a/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go b/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go index bbcaa50e6de..66c08e1c86c 100644 --- a/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go +++ b/pkg/plugin/generated/backupitemaction/v2/BackupItemAction.pb.go @@ -102,6 +102,7 @@ type ExecuteResponse struct { Item []byte `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"` AdditionalItems []*generated.ResourceIdentifier `protobuf:"bytes,2,rep,name=additionalItems,proto3" json:"additionalItems,omitempty"` OperationID string `protobuf:"bytes,3,opt,name=operationID,proto3" json:"operationID,omitempty"` + ItemsToUpdate []*generated.ResourceIdentifier `protobuf:"bytes,4,rep,name=itemsToUpdate,proto3" json:"itemsToUpdate,omitempty"` } func (x *ExecuteResponse) Reset() { @@ -157,6 +158,13 @@ func (x *ExecuteResponse) GetOperationID() string { return "" } +func (x *ExecuteResponse) GetItemsToUpdate() []*generated.ResourceIdentifier { + if x != nil { + return x.ItemsToUpdate + } + return nil +} + type BackupItemActionAppliesToRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -438,7 +446,7 @@ var file_backupitemaction_v2_BackupItemAction_proto_rawDesc = []byte{ 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x22, 0x90, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x22, 0xd5, 0x01, 0x0a, 0x0f, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x69, 0x74, 0x65, 0x6d, 0x12, 0x47, 0x0a, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, @@ -447,63 +455,67 @@ var file_backupitemaction_v2_BackupItemAction_proto_rawDesc = []byte{ 0x72, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, - 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, - 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, - 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x22, 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x73, - 0x0a, 0x1f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, - 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x22, 0x5c, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, + 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x43, 0x0a, 0x0d, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x54, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x0d, 0x69, 0x74, 0x65, 0x6d, + 0x73, 0x54, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x3a, 0x0a, 0x20, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, + 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, + 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, + 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x22, 0x6c, 0x0a, 0x21, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, + 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, + 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x52, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, + 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x22, 0x73, 0x0a, 0x1f, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, - 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x22, 0x71, 0x0a, 0x1d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, - 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x62, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x32, 0xbc, 0x02, 0x0a, 0x10, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, - 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x09, 0x41, 0x70, 0x70, - 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, - 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, - 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x76, - 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x12, 0x12, - 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, - 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, 0x20, + 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, + 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x22, 0x5c, 0x0a, 0x20, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x08, + 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x08, 0x70, 0x72, + 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x71, 0x0a, 0x1d, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x12, + 0x20, 0x0a, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x44, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x06, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x32, 0xbc, 0x02, 0x0a, 0x10, 0x42, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x58, + 0x0a, 0x09, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x12, 0x24, 0x2e, 0x76, 0x32, + 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x25, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, + 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x73, 0x54, 0x6f, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x07, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x65, 0x12, 0x12, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x08, + 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x23, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, - 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x21, 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x61, - 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, 0x6e, 0x7a, 0x75, 0x2f, 0x76, - 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, + 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x06, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x21, 0x2e, + 0x76, 0x32, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x74, 0x65, 0x6d, 0x41, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x49, 0x5a, 0x47, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x74, 0x61, + 0x6e, 0x7a, 0x75, 0x2f, 0x76, 0x65, 0x6c, 0x65, 0x72, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, + 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2f, + 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x69, 0x74, 0x65, 0x6d, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x2f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -534,21 +546,22 @@ var file_backupitemaction_v2_BackupItemAction_proto_goTypes = []interface{}{ } var file_backupitemaction_v2_BackupItemAction_proto_depIdxs = []int32{ 7, // 0: v2.ExecuteResponse.additionalItems:type_name -> generated.ResourceIdentifier - 8, // 1: v2.BackupItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector - 9, // 2: v2.BackupItemActionProgressResponse.progress:type_name -> generated.OperationProgress - 2, // 3: v2.BackupItemAction.AppliesTo:input_type -> v2.BackupItemActionAppliesToRequest - 0, // 4: v2.BackupItemAction.Execute:input_type -> v2.ExecuteRequest - 4, // 5: v2.BackupItemAction.Progress:input_type -> v2.BackupItemActionProgressRequest - 6, // 6: v2.BackupItemAction.Cancel:input_type -> v2.BackupItemActionCancelRequest - 3, // 7: v2.BackupItemAction.AppliesTo:output_type -> v2.BackupItemActionAppliesToResponse - 1, // 8: v2.BackupItemAction.Execute:output_type -> v2.ExecuteResponse - 5, // 9: v2.BackupItemAction.Progress:output_type -> v2.BackupItemActionProgressResponse - 10, // 10: v2.BackupItemAction.Cancel:output_type -> google.protobuf.Empty - 7, // [7:11] is the sub-list for method output_type - 3, // [3:7] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 7, // 1: v2.ExecuteResponse.itemsToUpdate:type_name -> generated.ResourceIdentifier + 8, // 2: v2.BackupItemActionAppliesToResponse.ResourceSelector:type_name -> generated.ResourceSelector + 9, // 3: v2.BackupItemActionProgressResponse.progress:type_name -> generated.OperationProgress + 2, // 4: v2.BackupItemAction.AppliesTo:input_type -> v2.BackupItemActionAppliesToRequest + 0, // 5: v2.BackupItemAction.Execute:input_type -> v2.ExecuteRequest + 4, // 6: v2.BackupItemAction.Progress:input_type -> v2.BackupItemActionProgressRequest + 6, // 7: v2.BackupItemAction.Cancel:input_type -> v2.BackupItemActionCancelRequest + 3, // 8: v2.BackupItemAction.AppliesTo:output_type -> v2.BackupItemActionAppliesToResponse + 1, // 9: v2.BackupItemAction.Execute:output_type -> v2.ExecuteResponse + 5, // 10: v2.BackupItemAction.Progress:output_type -> v2.BackupItemActionProgressResponse + 10, // 11: v2.BackupItemAction.Cancel:output_type -> google.protobuf.Empty + 8, // [8:12] is the sub-list for method output_type + 4, // [4:8] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_backupitemaction_v2_BackupItemAction_proto_init() } diff --git a/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto b/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto index 5f7bfe7d33a..366141dcad8 100644 --- a/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto +++ b/pkg/plugin/proto/backupitemaction/v2/BackupItemAction.proto @@ -16,6 +16,7 @@ message ExecuteResponse { bytes item = 1; repeated generated.ResourceIdentifier additionalItems = 2; string operationID = 3; + repeated generated.ResourceIdentifier itemsToUpdate = 4; } service BackupItemAction { diff --git a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go index a4a70234e75..4c9739b3003 100644 --- a/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go +++ b/pkg/plugin/velero/backupitemaction/v2/backup_item_action.go @@ -29,6 +29,9 @@ import ( // BackupItemAction is an actor that performs an operation on an individual item being backed up. type BackupItemAction interface { + // Name returns the name of this BIA + Name() string + // AppliesTo returns information about which resources this action should be invoked for. // A BackupItemAction's Execute function will only be invoked on items that match the returned // selector. A zero-valued ResourceSelector matches all resources. @@ -37,8 +40,13 @@ type BackupItemAction interface { // Execute allows the BackupItemAction to perform arbitrary logic with the item being backed up, // including mutating the item itself prior to backup. The item (unmodified or modified) // should be returned, along with an optional slice of ResourceIdentifiers specifying - // additional related items that should be backed up. - Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) + // additional related items that should be backed up, an optional operationID for actions which + // initiate asynchronous actions, and an optional slice of ResourceIdentifiers for any items + // which need to be updated in the Backup after async action processing. This last field will be ignored + // if operationID is empty, and should not be filled in unless the resource must be updated in the + // backup after async operations complete (i.e. some of the item's kubernetes metadata will be updated + // during the asynch operation which will be required during restore) + Execute(item runtime.Unstructured, backup *api.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) // Progress allows the BackupItemAction to report on progress of an asynchronous action. // For the passed-in operation, the plugin will return an OperationProgress struct, indicating diff --git a/pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go b/pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go index 63be81d4d66..42644247787 100644 --- a/pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go +++ b/pkg/plugin/velero/mocks/backupitemaction/v2/BackupItemAction.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery v2.16.0. DO NOT EDIT. package v2 @@ -67,7 +67,7 @@ func (_m *BackupItemAction) Cancel(operationID string, backup *v1.Backup) error } // Execute provides a mock function with given fields: item, backup -func (_m *BackupItemAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, error) { +func (_m *BackupItemAction) Execute(item runtime.Unstructured, backup *v1.Backup) (runtime.Unstructured, []velero.ResourceIdentifier, string, []velero.ResourceIdentifier, error) { ret := _m.Called(item, backup) var r0 runtime.Unstructured @@ -95,14 +95,37 @@ func (_m *BackupItemAction) Execute(item runtime.Unstructured, backup *v1.Backup r2 = ret.Get(2).(string) } - var r3 error - if rf, ok := ret.Get(3).(func(runtime.Unstructured, *v1.Backup) error); ok { + var r3 []velero.ResourceIdentifier + if rf, ok := ret.Get(3).(func(runtime.Unstructured, *v1.Backup) []velero.ResourceIdentifier); ok { r3 = rf(item, backup) } else { - r3 = ret.Error(3) + if ret.Get(3) != nil { + r3 = ret.Get(3).([]velero.ResourceIdentifier) + } } - return r0, r1, r2, r3 + var r4 error + if rf, ok := ret.Get(4).(func(runtime.Unstructured, *v1.Backup) error); ok { + r4 = rf(item, backup) + } else { + r4 = ret.Error(4) + } + + return r0, r1, r2, r3, r4 +} + +// Name provides a mock function with given fields: +func (_m *BackupItemAction) Name() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 } // Progress provides a mock function with given fields: operationID, backup @@ -125,3 +148,18 @@ func (_m *BackupItemAction) Progress(operationID string, backup *v1.Backup) (vel return r0, r1 } + +type mockConstructorTestingTNewBackupItemAction interface { + mock.TestingT + Cleanup(func()) +} + +// NewBackupItemAction creates a new instance of BackupItemAction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewBackupItemAction(t mockConstructorTestingTNewBackupItemAction) *BackupItemAction { + mock := &BackupItemAction{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/plugin/velero/shared.go b/pkg/plugin/velero/shared.go index 6316a1e0bff..636243eb75d 100644 --- a/pkg/plugin/velero/shared.go +++ b/pkg/plugin/velero/shared.go @@ -68,6 +68,20 @@ type ResourceIdentifier struct { Name string } +func (in *ResourceIdentifier) DeepCopy() *ResourceIdentifier { + if in == nil { + return nil + } + out := new(ResourceIdentifier) + in.DeepCopyInto(out) + return out +} + +func (in *ResourceIdentifier) DeepCopyInto(out *ResourceIdentifier) { + *out = *in + out.GroupResource = in.GroupResource +} + // OperationProgress describes progress of an asynchronous plugin operation. type OperationProgress struct { // True when the operation has completed, either successfully or with a failure