From 02b984debd0f7d6f902716b6572915f2deef1608 Mon Sep 17 00:00:00 2001 From: Anna Larch Date: Wed, 8 May 2024 20:38:51 +0200 Subject: [PATCH] fix(caldav): automatically delete outdated scheduling objects Signed-off-by: Anna Larch --- .../composer/composer/autoload_classmap.php | 2 + .../dav/composer/composer/autoload_static.php | 2 + .../DeleteOutdatedSchedulingObjects.php | 35 +++++++++++++++ apps/dav/lib/CalDAV/CalDavBackend.php | 38 ++++++++++++++++ .../lib/Migration/DeleteSchedulingObjects.php | 38 ++++++++++++++++ .../Version1004Date20170825134824.php | 1 + .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + .../lib/Controller/CheckSetupController.php | 3 ++ .../lib/SetupChecks/SchedulingTableSize.php | 44 +++++++++++++++++++ core/Application.php | 4 ++ core/Command/Db/AddMissingIndices.php | 12 ++++- core/js/setupchecks.js | 1 + lib/private/Repair.php | 2 + 14 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php create mode 100644 apps/dav/lib/Migration/DeleteSchedulingObjects.php create mode 100644 apps/settings/lib/SetupChecks/SchedulingTableSize.php diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index b8cf972a09c4d..d327a5e3a92bf 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -16,6 +16,7 @@ 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\PruneOutdatedSyncTokensJob' => $baseDir . '/../lib/BackgroundJob/PruneOutdatedSyncTokensJob.php', @@ -270,6 +271,7 @@ 'OCA\\DAV\\Migration\\BuildSocialSearchIndexBackgroundJob' => $baseDir . '/../lib/Migration/BuildSocialSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => $baseDir . '/../lib/Migration/CalDAVRemoveEmptyValue.php', 'OCA\\DAV\\Migration\\ChunkCleanup' => $baseDir . '/../lib/Migration/ChunkCleanup.php', + 'OCA\\DAV\\Migration\\DeleteSchedulingObjects' => $baseDir . '/../lib/Migration/DeleteSchedulingObjects.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => $baseDir . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => $baseDir . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\RegenerateBirthdayCalendars' => $baseDir . '/../lib/Migration/RegenerateBirthdayCalendars.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index ee586613ba086..5ee7b7d4326d7 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -31,6 +31,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\PruneOutdatedSyncTokensJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/PruneOutdatedSyncTokensJob.php', @@ -285,6 +286,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Migration\\BuildSocialSearchIndexBackgroundJob' => __DIR__ . '/..' . '/../lib/Migration/BuildSocialSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => __DIR__ . '/..' . '/../lib/Migration/CalDAVRemoveEmptyValue.php', 'OCA\\DAV\\Migration\\ChunkCleanup' => __DIR__ . '/..' . '/../lib/Migration/ChunkCleanup.php', + 'OCA\\DAV\\Migration\\DeleteSchedulingObjects' => __DIR__ . '/..' . '/../lib/Migration/DeleteSchedulingObjects.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => __DIR__ . '/..' . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => __DIR__ . '/..' . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\RegenerateBirthdayCalendars' => __DIR__ . '/..' . '/../lib/Migration/RegenerateBirthdayCalendars.php', diff --git a/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php b/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php new file mode 100644 index 0000000000000..fa53a8be4f02a --- /dev/null +++ b/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php @@ -0,0 +1,35 @@ +setInterval(23 * 60 * 60); + $this->setTimeSensitivity(self::TIME_INSENSITIVE); + } + + /** + * @param array $argument + */ + protected function run($argument): void { + $time = $this->time->getTime() - (60 * 60); + $this->calDavBackend->deleteOutdatedSchedulingObjects($time, 50000); + $this->logger->info("Removed outdated scheduling objects"); + } +} diff --git a/apps/dav/lib/CalDAV/CalDavBackend.php b/apps/dav/lib/CalDAV/CalDavBackend.php index ea332af3933fe..bae9427b655ec 100644 --- a/apps/dav/lib/CalDAV/CalDavBackend.php +++ b/apps/dav/lib/CalDAV/CalDavBackend.php @@ -2722,6 +2722,44 @@ public function deleteSchedulingObject($principalUri, $objectUri) { ->executeStatement(); } + /** + * Deletes all scheduling objects last modified before $modifiedBefore from the inbox collection. + * + * @param int $modifiedBefore + * @param int $limit + * @return void + */ + public function deleteOutdatedSchedulingObjects(int $modifiedBefore, int $limit): void { + $query = $this->db->getQueryBuilder(); + $query->select('id') + ->from('schedulingobjects') + ->where($query->expr()->lt('lastmodified', $query->createNamedParameter($modifiedBefore))) + ->setMaxResults($limit); + $result = $query->executeQuery(); + $count = $result->rowCount(); + if($count === 0) { + return; + } + $ids = array_map(static function (array $id) { + return (int)$id[0]; + }, $result->fetchAll(\PDO::FETCH_NUM)); + $result->closeCursor(); + + $numDeleted = 0; + $deleteQuery = $this->db->getQueryBuilder(); + $deleteQuery->delete('schedulingobjects') + ->where($deleteQuery->expr()->in('id', $deleteQuery->createParameter('ids'), IQueryBuilder::PARAM_INT_ARRAY)); + foreach(array_chunk($ids, 1000) as $chunk) { + $deleteQuery->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $numDeleted += $deleteQuery->executeStatement(); + } + + if($numDeleted === $limit) { + $this->logger->info("Deleted $limit scheduling objects, continuing with next batch"); + $this->deleteOutdatedSchedulingObjects($modifiedBefore, $limit); + } + } + /** * Creates a new scheduling object. This should land in a users' inbox. * diff --git a/apps/dav/lib/Migration/DeleteSchedulingObjects.php b/apps/dav/lib/Migration/DeleteSchedulingObjects.php new file mode 100644 index 0000000000000..3919236788bb3 --- /dev/null +++ b/apps/dav/lib/Migration/DeleteSchedulingObjects.php @@ -0,0 +1,38 @@ +info('Cleaning up old scheduling events'); + $time = $this->time->getTime() - (60 * 60); + $this->calDavBackend->deleteOutdatedSchedulingObjects($time, 50000); + if (!$this->jobList->has(DeleteOutdatedSchedulingObjects::class, null)) { + $output->info('Adding background job to delete old scheduling objects'); + $this->jobList->add(DeleteOutdatedSchedulingObjects::class, null); + } + } +} diff --git a/apps/dav/lib/Migration/Version1004Date20170825134824.php b/apps/dav/lib/Migration/Version1004Date20170825134824.php index a7cbaa78ef23e..7321bba62ff92 100644 --- a/apps/dav/lib/Migration/Version1004Date20170825134824.php +++ b/apps/dav/lib/Migration/Version1004Date20170825134824.php @@ -383,6 +383,7 @@ public function changeSchema(IOutput $output, \Closure $schemaClosure, array $op ]); $table->setPrimaryKey(['id']); $table->addIndex(['principaluri'], 'schedulobj_principuri_index'); + $table->addIndex(['lastmodified'], 'schedulobj_lastmodified_idx'); } if (!$schema->hasTable('cards_properties')) { diff --git a/apps/settings/composer/composer/autoload_classmap.php b/apps/settings/composer/composer/autoload_classmap.php index a1e0b7632ca5f..8f89ada8994ea 100644 --- a/apps/settings/composer/composer/autoload_classmap.php +++ b/apps/settings/composer/composer/autoload_classmap.php @@ -79,6 +79,7 @@ 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync' => $baseDir . '/../lib/SetupChecks/NeedsSystemAddressBookSync.php', 'OCA\\Settings\\SetupChecks\\PhpDefaultCharset' => $baseDir . '/../lib/SetupChecks/PhpDefaultCharset.php', 'OCA\\Settings\\SetupChecks\\PhpOutputBuffering' => $baseDir . '/../lib/SetupChecks/PhpOutputBuffering.php', + 'OCA\\Settings\\SetupChecks\\SchedulingTableSize' => $baseDir . '/../lib/SetupChecks/SchedulingTableSize.php', 'OCA\\Settings\\SetupChecks\\SupportedDatabase' => $baseDir . '/../lib/SetupChecks/SupportedDatabase.php', 'OCA\\Settings\\UserMigration\\AccountMigrator' => $baseDir . '/../lib/UserMigration/AccountMigrator.php', 'OCA\\Settings\\UserMigration\\AccountMigratorException' => $baseDir . '/../lib/UserMigration/AccountMigratorException.php', diff --git a/apps/settings/composer/composer/autoload_static.php b/apps/settings/composer/composer/autoload_static.php index 1eef91c5d91cf..77ec7c0fb612d 100644 --- a/apps/settings/composer/composer/autoload_static.php +++ b/apps/settings/composer/composer/autoload_static.php @@ -94,6 +94,7 @@ class ComposerStaticInitSettings 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync' => __DIR__ . '/..' . '/../lib/SetupChecks/NeedsSystemAddressBookSync.php', 'OCA\\Settings\\SetupChecks\\PhpDefaultCharset' => __DIR__ . '/..' . '/../lib/SetupChecks/PhpDefaultCharset.php', 'OCA\\Settings\\SetupChecks\\PhpOutputBuffering' => __DIR__ . '/..' . '/../lib/SetupChecks/PhpOutputBuffering.php', + 'OCA\\Settings\\SetupChecks\\SchedulingTableSize' => __DIR__ . '/..' . '/../lib/SetupChecks/SchedulingTableSize.php', 'OCA\\Settings\\SetupChecks\\SupportedDatabase' => __DIR__ . '/..' . '/../lib/SetupChecks/SupportedDatabase.php', 'OCA\\Settings\\UserMigration\\AccountMigrator' => __DIR__ . '/..' . '/../lib/UserMigration/AccountMigrator.php', 'OCA\\Settings\\UserMigration\\AccountMigratorException' => __DIR__ . '/..' . '/../lib/UserMigration/AccountMigratorException.php', diff --git a/apps/settings/lib/Controller/CheckSetupController.php b/apps/settings/lib/Controller/CheckSetupController.php index bf8de16a2b9b5..4de8e3c23896e 100644 --- a/apps/settings/lib/Controller/CheckSetupController.php +++ b/apps/settings/lib/Controller/CheckSetupController.php @@ -68,6 +68,7 @@ use OCA\Settings\SetupChecks\PhpDefaultCharset; use OCA\Settings\SetupChecks\PhpOutputBuffering; use OCA\Settings\SetupChecks\SupportedDatabase; +use OCA\Settings\SetupChecks\SchedulingTableSize; use OCP\App\IAppManager; use OCP\AppFramework\Controller; use OCP\AppFramework\Http\DataDisplayResponse; @@ -902,6 +903,7 @@ public function check() { $supportedDatabases = new SupportedDatabase($this->l10n, $this->connection); $ldapInvalidUuids = new LdapInvalidUuids($this->appManager, $this->l10n, $this->serverContainer); $needsSystemAddressBookSync = new NeedsSystemAddressBookSync($this->config, $this->l10n); + $schedulingTableSize = new SchedulingTableSize($this->l10n, $this->connection); return new DataResponse( [ @@ -958,6 +960,7 @@ public function check() { 'temporaryDirectoryWritable' => $this->isTemporaryDirectoryWritable(), LdapInvalidUuids::class => ['pass' => $ldapInvalidUuids->run(), 'description' => $ldapInvalidUuids->description(), 'severity' => $ldapInvalidUuids->severity()], NeedsSystemAddressBookSync::class => ['pass' => $needsSystemAddressBookSync->run(), 'description' => $needsSystemAddressBookSync->description(), 'severity' => $needsSystemAddressBookSync->severity()], + SchedulingTableSize::class => ['pass' => $schedulingTableSize->run(), 'description' => $schedulingTableSize->description(), 'severity' => $schedulingTableSize->severity()], ] ); } diff --git a/apps/settings/lib/SetupChecks/SchedulingTableSize.php b/apps/settings/lib/SetupChecks/SchedulingTableSize.php new file mode 100644 index 0000000000000..ea1908215bffa --- /dev/null +++ b/apps/settings/lib/SetupChecks/SchedulingTableSize.php @@ -0,0 +1,44 @@ +l10n = $l10n; + $this->connection = $connection; + } + + public function description(): string { + return $this->l10n->t('You have more than 500 000 rows in the scheduling objects table. Please run the expensive repair jobs via occ maintenance:repair --include-expensive'); + } + + public function severity(): string { + return 'warning'; + } + + public function run(): bool { + $qb = $this->connection->getQueryBuilder(); + $qb->select($qb->func()->count('id')) + ->from('schedulingobjects'); + $query = $qb->executeQuery(); + $count = $query->fetchOne(); + $query->closeCursor(); + + return $count <= 500000; + } +} diff --git a/core/Application.php b/core/Application.php index 592e092966681..ee77e447b03ae 100644 --- a/core/Application.php +++ b/core/Application.php @@ -201,6 +201,10 @@ function (GenericEvent $event) use ($container) { if (!$table->hasIndex('schedulobj_principuri_index')) { $subject->addHintForMissingSubject($table->getName(), 'schedulobj_principuri_index'); } + $table = $schema->getTable('schedulingobjects'); + if (!$table->hasIndex('schedulobj_lastmodified_idx')) { + $subject->addHintForMissingSubject($table->getName(), 'schedulobj_lastmodified_idx'); + } } if ($schema->hasTable('properties')) { diff --git a/core/Command/Db/AddMissingIndices.php b/core/Command/Db/AddMissingIndices.php index 1044cea554780..78cde82ef8a5d 100644 --- a/core/Command/Db/AddMissingIndices.php +++ b/core/Command/Db/AddMissingIndices.php @@ -403,8 +403,18 @@ private function addCoreIndexes(OutputInterface $output, bool $dryRun): void { $output->writeln($sqlQueries); } $updated = true; - $output->writeln('schedulingobjects table updated successfully.'); } + if (!$table->hasIndex('schedulobj_lastmodified_idx')) { + $output->writeln('Adding schedulobj_lastmodified_idx index to the schedulingobjects table, this can take some time...'); + + $table->addIndex(['lastmodified'], 'schedulobj_lastmodified_idx'); + $sqlQueries = $this->connection->migrateToSchema($schema->getWrappedSchema(), $dryRun); + if ($dryRun && $sqlQueries !== null) { + $output->writeln($sqlQueries); + } + $updated = true; + } + $output->writeln('schedulingobjects table updated successfully.'); } $output->writeln('Check indices of the oc_properties table.'); diff --git a/core/js/setupchecks.js b/core/js/setupchecks.js index 3a836cd78c6d0..613aae1f084ce 100644 --- a/core/js/setupchecks.js +++ b/core/js/setupchecks.js @@ -543,6 +543,7 @@ OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\SupportedDatabase', messages) OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\LdapInvalidUuids', messages) OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync', messages) + OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\SchedulingTableSize', messages) } else { messages.push({ msg: t('core', 'Error occurred while checking server setup'), diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 99af19ddb79f4..04f7981db9639 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -81,6 +81,7 @@ use OC\Repair\RepairMimeTypes; use OC\Repair\SqliteAutoincrement; use OC\Template\JSCombiner; +use OCA\DAV\Migration\DeleteSchedulingObjects; use OCP\AppFramework\QueryException; use OCP\AppFramework\Utility\ITimeFactory; use OCP\Collaboration\Resources\IManager; @@ -227,6 +228,7 @@ public static function getExpensiveRepairSteps() { return [ new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()), \OC::$server->get(ValidatePhoneNumber::class), + \OC::$server->get(DeleteSchedulingObjects::class), ]; }