-
Notifications
You must be signed in to change notification settings - Fork 411
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
PageStorage: Log down the gc time for external pages (#5739)
ref #5697
- Loading branch information
1 parent
21a37ab
commit 6262e0a
Showing
11 changed files
with
384 additions
and
122 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
dbms/src/Storages/Page/V3/PageDirectory/ExternalIdsByNamespace.cpp
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| // Copyright 2022 PingCAP, Ltd. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #include <Storages/Page/PageDefines.h> | ||
| #include <Storages/Page/V3/PageDirectory/ExternalIdsByNamespace.h> | ||
|
|
||
| #include <mutex> | ||
|
|
||
| namespace DB::PS::V3 | ||
| { | ||
| void ExternalIdsByNamespace::addExternalIdUnlock(const std::shared_ptr<PageIdV3Internal> & external_id) | ||
| { | ||
| const NamespaceId & ns_id = external_id->high; | ||
| // create a new ExternalIds if the ns_id is not exists, else return | ||
| // the existing one. | ||
| auto [ns_iter, new_inserted] = ids_by_ns.try_emplace(ns_id, ExternalIds{}); | ||
| ns_iter->second.emplace_back(std::weak_ptr<PageIdV3Internal>(external_id)); | ||
| } | ||
|
|
||
| void ExternalIdsByNamespace::addExternalId(const std::shared_ptr<PageIdV3Internal> & external_id) | ||
| { | ||
| std::unique_lock map_guard(mu); | ||
| addExternalIdUnlock(external_id); | ||
| } | ||
|
|
||
| std::set<PageId> ExternalIdsByNamespace::getAliveIds(NamespaceId ns_id) const | ||
| { | ||
| // Now we assume a lock among all NamespaceIds is good enough. | ||
| std::unique_lock map_guard(mu); | ||
|
|
||
| std::set<PageId> valid_external_ids; | ||
| auto ns_iter = ids_by_ns.find(ns_id); | ||
| if (ns_iter == ids_by_ns.end()) | ||
| return valid_external_ids; | ||
|
|
||
| // Only scan the given `ns_id` | ||
| auto & external_ids = ns_iter->second; | ||
| for (auto iter = external_ids.begin(); iter != external_ids.end(); /*empty*/) | ||
| { | ||
| if (auto holder = iter->lock(); holder == nullptr) | ||
| { | ||
| // the external id has been removed from `PageDirectory`, | ||
| // cleanup the invalid weak_ptr | ||
| iter = external_ids.erase(iter); | ||
| continue; | ||
| } | ||
| else | ||
| { | ||
| valid_external_ids.emplace(holder->low); | ||
| ++iter; | ||
| } | ||
| } | ||
| // No valid external pages in this `ns_id` | ||
| if (valid_external_ids.empty()) | ||
| { | ||
| valid_external_ids.erase(ns_id); | ||
| } | ||
| return valid_external_ids; | ||
| } | ||
|
|
||
| void ExternalIdsByNamespace::unregisterNamespace(NamespaceId ns_id) | ||
| { | ||
| std::unique_lock map_guard(mu); | ||
| // free all weak_ptrs of this namespace | ||
| ids_by_ns.erase(ns_id); | ||
| } | ||
| } // namespace DB::PS::V3 |
56 changes: 56 additions & 0 deletions
56
dbms/src/Storages/Page/V3/PageDirectory/ExternalIdsByNamespace.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| // Copyright 2022 PingCAP, Ltd. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <Common/nocopyable.h> | ||
| #include <Storages/Page/PageDefines.h> | ||
|
|
||
| #include <unordered_map> | ||
|
|
||
| namespace DB::PS::V3 | ||
| { | ||
|
|
||
| // A thread-safe class to manage external ids. | ||
| // Manage all external ids by NamespaceId. | ||
| class ExternalIdsByNamespace | ||
| { | ||
| public: | ||
| ExternalIdsByNamespace() = default; | ||
|
|
||
| // Add a external ids | ||
| void addExternalId(const std::shared_ptr<PageIdV3Internal> & external_id); | ||
| // non thread-safe version, only for restore | ||
| void addExternalIdUnlock(const std::shared_ptr<PageIdV3Internal> & external_id); | ||
|
|
||
| // Get all alive external ids of given `ns_id` | ||
| // Will also cleanup the invalid external ids. | ||
| std::set<PageId> getAliveIds(NamespaceId ns_id) const; | ||
|
|
||
| // After table dropped, the `getAliveIds` with specified | ||
| // `ns_id` will not be cleaned. We need this method to | ||
| // cleanup all external id ptrs. | ||
| void unregisterNamespace(NamespaceId ns_id); | ||
|
|
||
| DISALLOW_COPY_AND_MOVE(ExternalIdsByNamespace); | ||
|
|
||
| private: | ||
| mutable std::mutex mu; | ||
| // Only store weak_ptrs. The weak_ptrs will be invalid after the external id | ||
| // in PageDirectory get removed. | ||
| using ExternalIds = std::list<std::weak_ptr<PageIdV3Internal>>; | ||
| using NamespaceMap = std::unordered_map<NamespaceId, ExternalIds>; | ||
| mutable NamespaceMap ids_by_ns; | ||
| }; | ||
| } // namespace DB::PS::V3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.